DefStream 0.1.2__py3-none-any.whl

DefStream/Inspector/Extensions/Console.py

@@ -0,0 +1,396 @@
+from hashlib import sha256
+from hmac import compare_digest
+from html import escape
+from importlib.resources import files
+from json import (
+	dumps,
+	loads,
+)
+from logging import getLogger
+from math import isfinite
+from re import sub
+from threading import BoundedSemaphore
+from time import monotonic
+from typing import (
+	Callable,
+	Optional,
+	Protocol,
+	TYPE_CHECKING,
+)
+from urllib.parse import urlsplit
+
+from ...Broker.Connection import (
+	Connection,
+	Requestable,
+	TaskSender,
+)
+from ...Broker.Errors import RemoteError
+from ...Worker.Message import (
+	message_body,
+	request,
+	send,
+)
+from ..Extension import Extension
+from ..Response import Response
+from .Specifications import Specifications
+
+if TYPE_CHECKING:
+	from ..Inspector import Inspector
+
+__all__ = (
+	'Console',
+)
+
+logger = getLogger(__name__)
+
+
+class Producer(
+	Connection,
+	TaskSender,
+	Protocol,
+):
+	pass
+
+
+def json_response(
+	status: str,
+	value: dict,
+) -> Response:
+	return Response(
+		status,
+		dumps(
+			value,
+			ensure_ascii=False,
+			allow_nan=False,
+		).encode('utf-8'),
+		'application/json; charset=utf-8',
+		(
+			(
+				'X-Content-Type-Options',
+				'nosniff',
+			),
+		),
+	)
+
+
+class Console(Extension):
+	methods = (
+		'GET',
+		'HEAD',
+		'POST',
+	)
+
+	def __init__(
+		self,
+		path: str,
+		*,
+		broker: Callable[[], Producer],
+		token: Optional[str] = None,
+		timeout: float = 5,
+		title: str = 'DefStream',
+		description: Optional[str] = None,
+	):
+		super().__init__(path)
+		if not callable(broker):
+			raise TypeError('Console broker must be a factory creating a new producer connection')
+		if token is not None and (not isinstance(
+			token,
+			str,
+		) or len(token) < 16):
+			raise ValueError('Console token must contain at least 16 characters')
+		if isinstance(
+			timeout,
+			bool,
+		) or not isfinite(timeout) or timeout <= 0:
+			raise ValueError('Console RPC timeout must be finite and positive')
+		self.broker = broker
+		if not isinstance(
+			title,
+			str,
+		) or not title:
+			raise ValueError('Console title must be a nonempty string')
+		if description is not None and not isinstance(
+			description,
+			str,
+		):
+			raise TypeError('Console description must be a string or None')
+		self.title = title
+		self.description = description
+		self.token_digest = sha256(token.encode('utf-8')).digest() if token is not None else None
+		self.timeout = timeout
+		self.slots = BoundedSemaphore(4)
+
+	def __call__(
+		self,
+		inspector: 'Inspector',
+	) -> Response:
+		page = files(__package__).joinpath('Console.html').read_bytes()
+		values = {
+			'title': escape(self.title),
+			'description': escape(self.description) if self.description is not None else '',
+		}
+		page = sub(
+			r'\{\{console_(title|description)\}\}',
+			lambda match: values[match[1]],
+			page.decode('utf-8'),
+		).encode('utf-8')
+		if self.token_digest is None:
+			page = page.replace(
+				b'data-auth-required="true"',
+				b'data-auth-required="false"',
+			)
+		return Response(
+			'200 OK',
+			page,
+			'text/html; charset=utf-8',
+			(
+				(
+					'Content-Security-Policy',
+					"default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; connect-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'none'",
+				),
+				(
+					'X-Content-Type-Options',
+					'nosniff',
+				),
+				(
+					'Referrer-Policy',
+					'no-referrer',
+				),
+			),
+		)
+
+	def handle(
+		self,
+		inspector: 'Inspector',
+		environ: dict,
+	) -> Response:
+		method = environ['REQUEST_METHOD']
+		if method in (
+			'GET',
+			'HEAD',
+		) and environ.get('HTTP_ACCEPT') != 'application/json':
+			return self(inspector)
+		authorization = environ.get(
+			'HTTP_AUTHORIZATION',
+			'',
+		)
+		if self.token_digest is not None and (not authorization.startswith('Bearer ') or not compare_digest(
+			sha256(authorization[7:].encode('utf-8')).digest(),
+			self.token_digest,
+		)):
+			logger.warning('Inspector Console rejected an unauthorized request')
+			return json_response(
+				'401 Unauthorized',
+				{'error': 'A valid Bearer token is required'},
+			)
+		origin = environ.get('HTTP_ORIGIN')
+		if origin is not None:
+			try:
+				parsed = urlsplit(origin)
+				valid_origin = parsed.scheme in (
+					'http',
+					'https',
+				) and parsed.netloc == environ.get('HTTP_HOST')
+			except ValueError:
+				valid_origin = False
+			if not valid_origin:
+				logger.warning('Inspector Console rejected a cross-origin request')
+				return json_response(
+					'403 Forbidden',
+					{'error': 'Cross-origin requests are not allowed'},
+				)
+		if not self.slots.acquire(blocking=False):
+			return json_response(
+				'429 Too Many Requests',
+				{'error': 'Too many concurrent console requests'},
+			)
+		try:
+			if method in (
+				'GET',
+				'HEAD',
+			):
+				return self.catalog(inspector)
+			return self.execute(
+				inspector,
+				environ,
+			)
+		except Exception:
+			logger.exception('Inspector Console request failed')
+			return json_response(
+				'500 Internal Server Error',
+				{'error': 'Inspector console failed; consult the worker logs'},
+			)
+		finally:
+			self.slots.release()
+
+	def catalog(
+		self,
+		inspector: 'Inspector',
+	) -> Response:
+		producer = self.broker()
+		try:
+			if inspector.worker is not None and producer is inspector.worker.broker:
+				raise ValueError('Console requires a separate producer, not the Worker connection')
+			rpc = isinstance(
+				producer,
+				Requestable,
+			)
+			specifications = next(
+				(
+					extension for extension in inspector.exts.values()
+					if isinstance(
+						extension,
+						Specifications,
+					)
+				),
+				None,
+			)
+			if specifications is None:
+				specifications = Specifications(self.path)
+			document = specifications.document(inspector)
+			return json_response(
+				'200 OK',
+				{
+					'specification': document,
+					'rpc': rpc,
+					'timeout': self.timeout,
+				},
+			)
+		finally:
+			if inspector.worker is None or producer is not inspector.worker.broker:
+				producer.close()
+
+	def execute(
+		self,
+		inspector: 'Inspector',
+		environ: dict,
+	) -> Response:
+		worker = inspector.worker
+		if worker is None or not worker.running.is_set() or worker.stopping.is_set() or worker.finished.is_set():
+			return json_response(
+				'503 Service Unavailable',
+				{'error': 'Worker is not running'},
+			)
+		if environ.get(
+			'CONTENT_TYPE',
+			'',
+		).split(';')[0].strip().lower() != 'application/json':
+			return json_response(
+				'415 Unsupported Media Type',
+				{'error': 'Use application/json'},
+			)
+		length = environ.get(
+			'CONTENT_LENGTH',
+			'',
+		)
+		if not length.isascii() or not length.isdecimal():
+			return json_response(
+				'411 Length Required',
+				{'error': 'A Content-Length is required'},
+			)
+		size = int(length)
+		if size > 65536:
+			return json_response(
+				'413 Payload Too Large',
+				{'error': 'Console requests must not exceed 64 KiB'},
+			)
+		raw = environ['wsgi.input'].read(size)
+		try:
+			if len(raw) != size:
+				raise ValueError('Incomplete request body')
+			payload = loads(raw)
+			if not isinstance(
+				payload,
+				dict,
+			) or set(payload) != {
+				'task',
+				'mode',
+				'body',
+			}:
+				raise ValueError('Expected task, mode and body fields')
+			if payload['mode'] not in (
+				'send',
+				'request',
+			):
+				raise ValueError('Mode must be send or request')
+			body = message_body(payload['body'])
+			dumps(
+				body,
+				allow_nan=False,
+			)
+			task = next(
+				(
+					task for task in worker.tasks if task.__name__ == payload['task']
+				),
+				None,
+			)
+			if task is None:
+				raise ValueError('Unknown registered task')
+			if sum(task.__name__ == payload['task'] for task in worker.tasks) != 1:
+				raise ValueError('Ambiguous registered task name')
+		except (
+			ValueError,
+			TypeError,
+			UnicodeDecodeError,
+		) as error:
+			logger.warning('Inspector Console rejected invalid input')
+			return json_response(
+				'400 Bad Request',
+				{'error': str(error)},
+			)
+		started = monotonic()
+		producer = self.broker()
+		if producer is worker.broker:
+			raise ValueError('Console requires a separate producer, not the Worker connection')
+		try:
+			if payload['mode'] == 'request' and not isinstance(
+				producer,
+				Requestable,
+			):
+				return json_response(
+					'501 Not Implemented',
+					{'error': 'This broker does not support RPC'},
+				)
+			producer.connect()
+			try:
+				if payload['mode'] == 'request':
+					result = request(
+						body,
+						task=task,
+						broker=producer,
+						timeout=self.timeout,
+					)
+				else:
+					result = send(
+						body,
+						task=task,
+						broker=producer,
+					)
+			except TimeoutError:
+				logger.warning('Inspector Console RPC timed out')
+				return json_response(
+					'504 Gateway Timeout',
+					{'error': 'RPC timed out; delivered work is not cancelled'},
+				)
+			except RemoteError as error:
+				logger.warning(
+					'Inspector Console received a remote task error: %s',
+					error.code,
+				)
+				return json_response(
+					'502 Bad Gateway',
+					{
+						'error': str(error),
+						'code': error.code,
+					},
+				)
+			return json_response(
+				'200 OK',
+				{
+					'mode': payload['mode'],
+					'result': result,
+					'elapsed': monotonic() - started,
+				},
+			)
+		finally:
+			producer.close()

DefStream/Inspector/Extensions/Dashboard.html

@@ -0,0 +1,206 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>{{dashboard_title}}</title>
+<style>
+:root { color-scheme: light; font: 14px/1.55 system-ui, -apple-system, sans-serif; color: #263343; background: #f7f8fa; }
+* { box-sizing: border-box; }
+body { margin: 0; }
+header { background: #fff; border-bottom: 1px solid #e1e5eb; }
+.heading, main { max-width: 1440px; margin: auto; padding: 24px 32px; }
+.heading { display: flex; align-items: center; justify-content: space-between; gap: 16px; }
+h1 { margin: 0; font-size: 22px; letter-spacing: -.6px; overflow-wrap: anywhere; }
+h2 { margin: 0; font-size: 15px; }
+p { margin: 6px 0 0; }
+.subtitle, .muted { color: #657184; font-size: 12px; }
+.subtitle:empty { display: none; }
+.badge { padding: 5px 10px; border: 1px solid #c6dfcf; border-radius: 5px; color: #286f40; background: #f0faf3; font-size: 11px; white-space: nowrap; }
+.toolbar { display: flex; align-items: center; justify-content: space-between; gap: 16px; margin-bottom: 20px; }
+#notice { margin: 0; font-size: 12px; color: #536374; }
+#notice[data-state="error"] { color: #ab303d; }
+button { padding: 8px 14px; border: 1px solid #cdd5df; border-radius: 5px; font: 600 12px system-ui, sans-serif; background: #fff; color: #38485b; cursor: pointer; white-space: nowrap; }
+button:hover { background: #edf3f8; }
+select { padding: 8px; border: 1px solid #cdd5df; border-radius: 5px; background: #fff; color: #38485b; }
+.controls { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; }
+:focus-visible { outline: 3px solid #89b8ff; outline-offset: 3px; }
+.cards { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 16px; margin-bottom: 24px; }
+.card, .panel { min-width: 0; background: #fff; border: 1px solid #e1e5eb; border-radius: 7px; }
+.card { padding: 20px; }
+.label { font-size: 12px; color: #657184; }
+.value { margin-top: 8px; font-size: 28px; font-weight: 650; line-height: 1.3; font-variant-numeric: tabular-nums; overflow-wrap: anywhere; }
+.rate { color: #27834c; }
+.failure { color: #b33b45; }
+.hint { color: #768295; font-size: 11px; margin-top: 8px; }
+.panel-heading { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 10px; padding: 18px 20px; border-bottom: 1px solid #e1e5eb; }
+.table-scroll { overflow: auto; }
+table { width: 100%; border-collapse: collapse; font-size: 12px; }
+th { background: #f7f9fb; color: #657184; font-weight: 600; white-space: nowrap; }
+th, td { padding: 13px 18px; text-align: right; border-bottom: 1px solid #edf0f4; font-variant-numeric: tabular-nums; }
+th:first-child, td:first-child { text-align: left; }
+td:first-child { max-width: 400px; overflow-wrap: anywhere; font: 12px/1.7 Consolas, monospace; }
+tr:last-child td { border-bottom: 0; }
+#empty { padding: 24px; color: #657184; font-size: 12px; }
+.notes { margin-top: 20px; color: #657184; font-size: 12px; }
+.stale .cards, .stale .panel { opacity: .55; }
+[hidden] { display: none !important; }
+@media(max-width: 800px) {
+  .heading, main { padding: 20px 16px; }
+  .cards { grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px; }
+  .card { padding: 16px; }
+  .value { font-size: 24px; }
+}
+@media(max-width: 420px) {
+  .cards { grid-template-columns: 1fr; }
+  .heading { align-items: start; }
+}
+</style>
+</head>
+<body>
+<header><div class="heading">
+<div><h1>{{dashboard_title}}</h1><p class="subtitle">{{dashboard_description}}</p></div>
+<span class="badge">LIVE DASHBOARD</span>
+</div></header>
+<main>
+<div class="toolbar">
+<p id="notice" role="status" aria-live="polite">Connecting to worker...</p>
+<div class="controls">
+<label for="window">View</label><select id="window"><option value="1">Last second</option><option value="60">Last minute</option></select>
+<button id="pause" type="button" aria-pressed="false">Pause updates</button>
+</div>
+</div>
+<div class="cards">
+<section class="card"><div class="label">Window completions</div><div id="rate" class="value rate">--</div><p id="window-hint" class="hint">Completed in the last fully elapsed second</p></section>
+<section class="card"><div class="label">In flight</div><div id="active" class="value">--</div><p class="hint">Submitted, not yet completed</p></section>
+<section class="card"><div class="label">Worker lifecycle</div><div id="state" class="value">--</div><p class="hint">Lifecycle only, not a health probe</p></section>
+<section class="card"><div class="label">Succeeded</div><div id="success" class="value">--</div><p class="hint">Since worker startup</p></section>
+<section class="card"><div class="label">Failed</div><div id="failure" class="value failure">--</div><p class="hint">Includes failed retry attempts</p></section>
+<section class="card"><div class="label">Uptime</div><div id="uptime" class="value">--</div><p class="hint">Since Metrics initialization</p></section>
+<section class="card"><div class="label">Window success / failure</div><div id="window-outcomes" class="value">--</div><p class="hint">Completed attempts in selected window</p></section>
+<section class="card"><div class="label">Window failure rate</div><div id="failure-rate" class="value">--</div><p class="hint">Failed / completed attempts</p></section>
+<section class="card"><div class="label">Window average time</div><div id="average" class="value">--</div><p class="hint">Includes pool wait and hooks</p></section>
+<section class="card"><div class="label">Window maximum time</div><div id="maximum" class="value">--</div><p class="hint">Slowest completion in selected window</p></section>
+<section class="card"><div class="label">Oldest in flight</div><div id="oldest" class="value">--</div><p class="hint">Elapsed since submission, including pool wait</p></section>
+</div>
+<section class="panel" aria-labelledby="tasks-title">
+<div class="panel-heading"><h2 id="tasks-title">Task activity</h2><span class="muted">Current rates &amp; cumulative totals</span></div>
+<div class="table-scroll"><table>
+<thead><tr><th scope="col">Task</th><th scope="col">In flight</th><th scope="col">Total success</th><th scope="col">Total failure</th><th id="rate-heading" scope="col">Tasks / sec</th><th scope="col">Window avg.</th><th scope="col">Window max.</th><th scope="col">Window success</th><th scope="col">Window failure</th><th scope="col">Failure rate</th><th scope="col">Oldest in flight</th></tr></thead>
+<tbody id="tasks"></tbody>
+</table></div>
+<p id="empty" hidden>No tasks registered.</p>
+</section>
+<p class="notes">Refreshes about every second. Windows end at the last fully elapsed monotonic second (less than one second behind live totals).
+Last minute counts actual completions in 60 seconds, not a multiplied per-second estimate. Startup windows include only data since startup.
+Counts represent execution attempts, not unique messages or confirmed ACKs. Average time includes pool wait and hooks.
+This page shows one worker only, with no persistent history.</p>
+</main>
+<script>
+'use strict';
+const el = id => document.getElementById(id);
+const number = new Intl.NumberFormat(undefined, { maximumFractionDigits: 2 });
+let latest = null;
+let timer = null;
+let pending = false;
+let paused = false;
+const milliseconds = value => number.format(value * 1000) + ' ms';
+const failureRate = window => window.success + window.failure ? number.format(window.failure / (window.success + window.failure) * 100) + '%' : '--';
+function duration(seconds) {
+  const total = Math.floor(seconds);
+  return [Math.floor(total / 3600), Math.floor(total % 3600 / 60), total % 60].map(value => String(value).padStart(2, '0')).join(':');
+}
+function render(snapshot) {
+  const seconds = el('window').value || '1';
+  const unit = seconds === '60' ? 'min' : 'sec';
+  const totals = { active: 0, success: 0, failure: 0 };
+  const window = { success: 0, failure: 0, duration: 0, max_duration: 0 };
+  let oldest = 0;
+  const rows = Object.entries(snapshot.tasks).map(([name, task]) => {
+    for (const key of Object.keys(totals)) totals[key] += task[key];
+    const period = task.windows[seconds];
+    for (const key of ['success', 'failure', 'duration']) window[key] += period[key];
+    window.max_duration = Math.max(window.max_duration, period.max_duration);
+    oldest = Math.max(oldest, task.oldest_active);
+    const count = period.success + period.failure;
+    const row = document.createElement('tr');
+    const values = [name, number.format(task.active), number.format(task.success), number.format(task.failure),
+      number.format(count), count ? milliseconds(period.duration / count) : '--',
+      count ? milliseconds(period.max_duration) : '--', number.format(period.success), number.format(period.failure),
+      failureRate(period), task.active ? milliseconds(task.oldest_active) : '--'];
+    values.forEach((value, index) => {
+      const cell = document.createElement('td');
+      if (index >= 4) cell.className = 'task-rate';
+      cell.textContent = value;
+      row.append(cell);
+    });
+    return row;
+  });
+  el('tasks').replaceChildren(...rows);
+  el('empty').hidden = rows.length > 0;
+  for (const key of Object.keys(totals)) el(key).textContent = number.format(totals[key]);
+  const count = window.success + window.failure;
+  el('rate').textContent = number.format(count) + ' /' + unit;
+  el('window-hint').textContent = 'Completed in the last fully elapsed ' + (seconds === '60' ? '60 seconds' : 'second');
+  el('rate-heading').textContent = 'Tasks / ' + unit;
+  el('window-outcomes').textContent = number.format(window.success) + ' / ' + number.format(window.failure);
+  el('failure-rate').textContent = failureRate(window);
+  el('average').textContent = count ? milliseconds(window.duration / count) : '--';
+  el('maximum').textContent = count ? milliseconds(window.max_duration) : '--';
+  el('oldest').textContent = totals.active ? milliseconds(oldest) : '--';
+  el('state').textContent = snapshot.running ? 'Running' : 'Not running';
+  el('uptime').textContent = duration(snapshot.uptime);
+  latest = snapshot;
+}
+function stale() {
+  latest = null;
+  document.body.classList.add('stale');
+  for (const id of ['rate', 'window-outcomes', 'failure-rate', 'average', 'maximum', 'oldest']) el(id).textContent = '--';
+  for (const cell of document.querySelectorAll('.task-rate')) cell.textContent = '--';
+}
+el('window').onchange = () => { if (latest) render(latest); };
+async function refresh() {
+  if (pending || paused || document.hidden) return;
+  pending = true;
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), 5000);
+  try {
+    const response = await fetch(window.location.pathname, {
+      headers: { Accept: 'application/json' }, credentials: 'omit', redirect: 'error', cache: 'no-store', signal: controller.signal
+    });
+    if (!response.ok) throw new Error('HTTP ' + response.status);
+    const snapshot = await response.json();
+    if (paused || document.hidden) return;
+    render(snapshot);
+    document.body.classList.remove('stale');
+    el('notice').dataset.state = 'ready';
+    el('notice').textContent = (paused ? 'Paused. Last updated ' : 'Live. Updated ') + new Date().toLocaleTimeString();
+  } catch (error) {
+    stale();
+    el('notice').dataset.state = 'error';
+    el('notice').textContent = 'Disconnected - displayed totals are stale. ' + (error.name === 'AbortError' ? 'Request timed out.' : error.message);
+  } finally {
+    clearTimeout(timeout);
+    pending = false;
+    if (!paused && !document.hidden) timer = setTimeout(refresh, 1000);
+  }
+}
+el('pause').onclick = () => {
+  paused = !paused;
+  clearTimeout(timer);
+  el('pause').textContent = paused ? 'Resume updates' : 'Pause updates';
+  el('pause').setAttribute('aria-pressed', String(paused));
+  if (paused) {
+    stale();
+    el('notice').textContent = 'Paused - displayed data is not updating.';
+  } else refresh();
+};
+document.addEventListener('visibilitychange', () => {
+  clearTimeout(timer);
+  if (document.hidden) stale();
+  if (!document.hidden) refresh();
+});
+refresh();
+</script>
+</body>
+</html>