somestuff-mqttwebstuff 0.1.4__py3-none-any.whl

somestuff/mqttwebstuff/__init__.py

@@ -0,0 +1,123 @@
+"""mqttwebstuff — live web view onto arbitrary MQTT streams.
+
+A small FastAPI app subscribes to an MQTT broker, runs every message through a
+*mapper plugin* (a plain Python file mounted into the container) and pushes the
+rendered result to all connected browsers via Server-Sent Events. The browser
+side is htmx + its SSE extension — no frontend build or framework.
+
+The mapper plugin decides *what* is shown (filter), *where* (panel/key) and
+*how* (Jinja2 template name); see :mod:`somestuff.mqttwebstuff.plugin_api` for the
+contract and ``plugins/oepnv_view.py`` for the reference plugin rendering the
+:mod:`oepnvstuff` departure board.
+
+Central logging setup lives here (loguru), mirroring oepnvstuff's setup. The
+modules in this package (and third-party libraries such as ``paho-mqtt`` and
+``uvicorn``) log via the stdlib ``logging`` module, so
+:func:`configure_logging` installs an intercept handler that funnels those
+records into loguru instead of rewriting every call site.
+"""
+
+import inspect
+import logging
+import os
+import sys
+from collections.abc import Callable
+from types import FrameType
+from typing import TYPE_CHECKING
+
+from loguru import logger as glogger
+from tabulate import tabulate
+
+if TYPE_CHECKING:
+    from loguru import Record
+
+__version__ = "0.1.4"
+
+__all__ = ["__version__", "configure_logging", "print_banner"]
+
+
+def _loguru_skiplog_filter(record: "Record") -> bool:
+    """Decide whether a loguru record should be emitted.
+
+    Args:
+        record: The loguru record about to be handled by a sink.
+
+    Returns:
+        ``False`` for records whose ``extra['skiplog']`` flag is truthy (they are
+        hidden), ``True`` otherwise.
+    """
+    return not record.get("extra", {}).get("skiplog", False)
+
+
+class _InterceptHandler(logging.Handler):
+    """Route stdlib ``logging`` records into loguru with correct call-site info.
+
+    Every module in this package uses ``logging.getLogger(...)``; installing this
+    on the root logger lets :func:`configure_logging` govern all of them (format,
+    level, colour) without touching the individual call sites.
+    """
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """Forward a single stdlib record to loguru.
+
+        Args:
+            record: The stdlib log record to re-emit through loguru, preserving
+                its level, message, exception info and originating call site.
+        """
+        # Map the stdlib level name to a loguru level, falling back to the number.
+        level: str | int
+        try:
+            level = glogger.level(record.levelname).name
+        except ValueError:
+            level = record.levelno
+        # Walk out of the logging module so {module}/{function}/{line} point at the
+        # real caller rather than at this handler (canonical loguru intercept recipe).
+        frame: FrameType | None = inspect.currentframe()
+        depth = 0
+        while frame and (depth == 0 or frame.f_code.co_filename == logging.__file__):
+            frame = frame.f_back
+            depth += 1
+        glogger.bind(classname=record.name).opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
+
+
+def configure_logging(
+    verbose: bool = False, loguru_filter: Callable[["Record"], bool] = _loguru_skiplog_filter
+) -> None:
+    """Configure a default ``loguru`` sink and funnel stdlib logging into it.
+
+    Args:
+        verbose: ``True`` logs at DEBUG, otherwise honours ``LOGURU_LEVEL`` (INFO
+            by default).
+        loguru_filter: Predicate deciding, per loguru record, whether it is emitted
+            by the sink. Defaults to :func:`_loguru_skiplog_filter`.
+    """
+    level_name: str = "DEBUG" if verbose else os.getenv("LOGURU_LEVEL", "INFO")
+    os.environ["LOGURU_LEVEL"] = level_name
+    glogger.remove()
+    logger_fmt: str = (
+        "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <8}</level> | <cyan>{module}</cyan>::<cyan>{extra[classname]}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>"
+    )
+    glogger.add(sys.stderr, level=level_name, format=logger_fmt, filter=loguru_filter)  # type: ignore[arg-type]
+    glogger.configure(extra={"classname": "None", "skiplog": False})
+    # Replace any stdlib handlers with the intercept so all ``logging`` output
+    # (this package's and third-party libraries') flows through loguru.
+    logging.basicConfig(handlers=[_InterceptHandler()], level=logging.getLevelNamesMapping()[level_name], force=True)
+
+
+def print_banner() -> None:
+    """Log the startup banner with version, build time and project links."""
+    startup_rows = [
+        ["version", __version__],
+        ["buildtime", os.getenv("BUILDTIME", "unknown")],
+        ["github", "https://github.com/vroomfondel/somestuff/tree/main/somestuff/mqttwebstuff"],
+        ["Docker Hub", "https://hub.docker.com/r/xomoxcc/somestuff"],
+    ]
+    table_str = tabulate(startup_rows, tablefmt="mixed_grid")
+    lines = table_str.split("\n")
+    table_width = len(lines[0])
+    title = "mqttweb live stream viewer starting up"
+    title_border = "┍" + "━" * (table_width - 2) + "┑"
+    title_row = "│ " + title.center(table_width - 4) + " │"
+    separator = lines[0].replace("┍", "┝").replace("┑", "┥").replace("┯", "┿")
+
+    glogger.opt(raw=True).info(f"\n{title_border}\n{title_row}\n{separator}\n{'\n'.join(lines[1:])}\n")

somestuff/mqttwebstuff/hub.py

@@ -0,0 +1,343 @@
+"""The view hub: last-value cache + SSE fan-out between MQTT and the browsers.
+
+The hub is the piece that turns a *non-retained* live stream into something a
+freshly opened browser tab can show immediately: every mapped message is
+rendered once and kept in an in-memory cache per ``(panel, key)``, so the
+initial page load renders the full current board server-side and the SSE
+stream afterwards only carries deltas.
+
+Threading model: paho-mqtt delivers messages on its own network thread;
+:meth:`ViewHub.submit` is the only thread-safe entry point and trampolines
+into the asyncio event loop (``call_soon_threadsafe``). Everything else —
+ingest, rendering, cache, subscriber queues — runs single-threaded on the
+event loop, so no locks are needed.
+
+Update granularity: one SSE event per *panel* (the panel's full body is
+re-rendered from cache and swapped via htmx ``sse-swap``). At the message
+rates of a poll-based stream this is trivially cheap and makes ordering,
+replacement and expiry correct by construction. Only when a message creates a
+panel the initial page did not know about is the whole board re-sent once
+(event ``board``) so the new section materializes without a reload.
+"""
+
+import asyncio
+import html
+import json
+import logging
+import re
+import time
+import unicodedata
+from dataclasses import dataclass
+from typing import Any
+
+import jinja2
+
+from somestuff.mqttwebstuff.plugin_api import GENERIC_TEMPLATE, LoadedPlugin, ViewEvent
+
+logger = logging.getLogger(__name__)
+
+# German umlauts/eszett → ASCII digraphs, applied before the generic
+# accent-stripping so "Straße" slugs to "strasse", not "strae".
+_UMLAUTS = str.maketrans({"ä": "ae", "ö": "oe", "ü": "ue", "Ä": "Ae", "Ö": "Oe", "Ü": "Ue", "ß": "ss"})
+
+
+def anchor_slug(text: str) -> str:
+    """Turn a label/topic into a stable, URL-fragment-safe anchor id part.
+
+    Deterministic, so deep links (``/#g-departures-ellerbek-moordamm``) stay
+    valid across repaints and restarts as long as the label itself is stable.
+
+    Args:
+        text: Display label or topic path.
+
+    Returns:
+        Lowercased ASCII slug (never empty).
+    """
+    text = unicodedata.normalize("NFKD", text.translate(_UMLAUTS)).encode("ascii", "ignore").decode()
+    return re.sub(r"[^a-zA-Z0-9]+", "-", text).strip("-").lower() or "x"
+
+
+# Per-subscriber queue depth; a browser that cannot drain this many panel
+# updates gets the oldest ones dropped (the next update repaints the panel
+# fully anyway, so drops only cost intermediate states).
+_QUEUE_MAXSIZE = 256
+
+
+@dataclass(slots=True)
+class _Entry:
+    """One cached board item: pre-rendered HTML plus grouping/ordering/expiry."""
+
+    html: str
+    sort: str
+    group: str
+    expires: float | None
+
+
+def decode_payload(raw: str | bytes) -> Any:
+    """Decode a raw MQTT payload: JSON when it parses, the string otherwise.
+
+    Args:
+        raw: The payload as delivered by the broker.
+
+    Returns:
+        ``dict``/``list``/scalar for valid JSON, otherwise the decoded string.
+    """
+    text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
+    stripped = text.strip()
+    if stripped.startswith(("{", "[")):
+        try:
+            return json.loads(stripped)
+        except ValueError:
+            pass
+    return text
+
+
+class ViewHub:
+    """Cache + broadcast hub; single-threaded on the asyncio event loop.
+
+    Args:
+        env: Jinja2 environment used to render item templates.
+        plugin: The mapper plugin deciding filter/panel/template per message.
+    """
+
+    def __init__(self, env: jinja2.Environment, plugin: LoadedPlugin) -> None:
+        self._env = env
+        self._plugin = plugin
+        # Declared panels exist from the start (stable board layout even while
+        # empty); undeclared ones are appended when their first item arrives.
+        self._panels: dict[str, dict[str, _Entry]] = {name: {} for name in plugin.panels}
+        self._subscribers: set[asyncio.Queue[tuple[str, str]]] = set()
+        self._loop: asyncio.AbstractEventLoop | None = None
+
+    @property
+    def plugin(self) -> LoadedPlugin:
+        """The mapper plugin this hub runs."""
+        return self._plugin
+
+    def attach_loop(self, loop: asyncio.AbstractEventLoop) -> None:
+        """Bind the hub to the running event loop (called from app startup).
+
+        Args:
+            loop: The loop all ingest/broadcast work must run on.
+        """
+        self._loop = loop
+
+    # ── thread-safe entry point (paho network thread) ───────────────────────
+
+    def submit(self, topic: str, raw: str | bytes) -> None:
+        """Hand one MQTT message over to the event loop; safe from any thread.
+
+        Messages arriving before the loop is attached (or after shutdown) are
+        dropped — with a non-retained live stream the next cycle re-delivers
+        the current state anyway.
+
+        Args:
+            topic: Full MQTT topic of the message.
+            raw: Raw payload.
+        """
+        loop = self._loop
+        if loop is None or loop.is_closed():
+            return
+        payload = decode_payload(raw)
+        try:
+            loop.call_soon_threadsafe(self.ingest, topic, payload)
+        except RuntimeError:
+            # Loop shut down between the check and the call — shutdown race, drop.
+            pass
+
+    # ── event-loop side ─────────────────────────────────────────────────────
+
+    def ingest(self, topic: str, payload: Any) -> None:
+        """Map, render and cache one message, then broadcast the delta.
+
+        A mapper may return one :class:`ViewEvent` or a sequence of them (e.g.
+        a leaf plus synthesized ancestor rows for a tree view); affected panels
+        are each repainted once. A crashing mapper or template must never take
+        the stream down: errors are logged and the message is dropped.
+
+        Args:
+            topic: Full MQTT topic of the message.
+            payload: Decoded payload (see :func:`decode_payload`).
+        """
+        try:
+            result = self._plugin.map_message(topic, payload)
+        except Exception:
+            logger.exception(f"mapper failed for topic {topic}")
+            return
+        if result is None:
+            return
+        events = [result] if isinstance(result, ViewEvent) else list(result)
+        if not events:
+            return
+
+        new_panel = False
+        touched: set[str] = set()
+        for event in events:
+            try:
+                rendered = self._render_item(event, topic)
+            except Exception:
+                logger.exception(f"template {event.template or GENERIC_TEMPLATE} failed for topic {topic}")
+                continue
+            new_panel = new_panel or event.panel not in self._panels
+            panel = self._panels.setdefault(event.panel, {})
+            panel[event.key] = _Entry(
+                html=rendered,
+                sort=event.sort or event.key,
+                group=event.group,
+                expires=(time.monotonic() + event.ttl) if event.ttl is not None else None,
+            )
+            touched.add(event.panel)
+
+        if new_panel:
+            self._broadcast("board", self.render_board())
+        else:
+            for name in touched:
+                self._broadcast(f"panel:{name}", self.render_panel_body(name))
+
+    def _render_item(self, event: ViewEvent, topic: str) -> str:
+        """Render one item to its HTML fragment.
+
+        Args:
+            event: The mapped view event.
+            topic: Originating topic, exposed to the template as ``topic``.
+
+        Returns:
+            The rendered fragment.
+        """
+        template = self._env.get_template(event.template or GENERIC_TEMPLATE)
+        return template.render(data=event.data, topic=topic, key=event.key, panel=event.panel, title=event.title)
+
+    def render_panel_body(self, name: str) -> str:
+        """Render a panel's body: items in sort order, or a placeholder.
+
+        Items carrying a ``group`` label are clustered under a sub-heading per
+        label (labels sorted alphabetically, items within a group by their
+        ``sort`` key); ungrouped items render first.
+
+        Args:
+            name: Panel name.
+
+        Returns:
+            HTML for the panel's inner container.
+        """
+        entries = self._panels.get(name, {})
+        if not entries:
+            return '<p class="mqttweb-empty">– keine Daten –</p>'
+
+        ungrouped = sorted((e for e in entries.values() if not e.group), key=lambda e: e.sort)
+        groups: dict[str, list[_Entry]] = {}
+        for entry in entries.values():
+            if entry.group:
+                groups.setdefault(entry.group, []).append(entry)
+
+        parts: list[str] = [e.html for e in ungrouped]
+        for label in sorted(groups):
+            items = "\n".join(e.html for e in sorted(groups[label], key=lambda e: e.sort))
+            # Stable anchor id so each group (e.g. one stop) is deep-linkable
+            # via /#g-<panel>-<label>; the heading doubles as the self-link.
+            anchor = f"g-{anchor_slug(name)}-{anchor_slug(label)}"
+            parts.append(
+                f'<div class="mqttweb-group-box" id="{anchor}">'
+                f'<h3 class="mqttweb-group"><a class="mqttweb-anchor" href="#{anchor}">{html.escape(label)}</a></h3>'
+                f"\n{items}</div>"
+            )
+        return "\n".join(parts)
+
+    def render_board(self) -> str:
+        """Render the whole board (all panels) from cache.
+
+        Declared panels keep their declared order; panels created at runtime
+        follow alphabetically.
+
+        Returns:
+            HTML for the board container.
+        """
+        declared = [name for name in self._plugin.panels if name in self._panels]
+        extra = sorted(name for name in self._panels if name not in self._plugin.panels)
+        sections: list[str] = []
+        for name in declared + extra:
+            # An empty declared heading renders the panel "plain": no title and
+            # no panel chrome — its items (e.g. group boxes) stand on their own.
+            heading = self._plugin.panels.get(name, name)
+            css = "mqttweb-panel" if heading else "mqttweb-panel mqttweb-panel-plain"
+            title_html = f"<h2>{html.escape(heading)}</h2>" if heading else ""
+            sections.append(
+                f'<section class="{css}" id="panel-{html.escape(name, quote=True)}">{title_html}'
+                f'<div class="mqttweb-panel-body" sse-swap="panel:{html.escape(name, quote=True)}">'
+                f"{self.render_panel_body(name)}</div></section>"
+            )
+        return "\n".join(sections)
+
+    # ── expiry ──────────────────────────────────────────────────────────────
+
+    def sweep_once(self, now: float | None = None) -> list[str]:
+        """Drop expired items and broadcast every panel that changed.
+
+        Args:
+            now: Reference monotonic time (defaults to ``time.monotonic()``;
+                injectable for tests).
+
+        Returns:
+            Names of the panels that lost at least one item.
+        """
+        now = time.monotonic() if now is None else now
+        changed: list[str] = []
+        for name, entries in self._panels.items():
+            expired = [key for key, entry in entries.items() if entry.expires is not None and entry.expires <= now]
+            if not expired:
+                continue
+            for key in expired:
+                del entries[key]
+            changed.append(name)
+            self._broadcast(f"panel:{name}", self.render_panel_body(name))
+        return changed
+
+    async def sweeper(self, interval: float = 5.0) -> None:
+        """Run :meth:`sweep_once` forever; cancelled on app shutdown.
+
+        Args:
+            interval: Seconds between sweeps.
+        """
+        while True:
+            await asyncio.sleep(interval)
+            self.sweep_once()
+
+    # ── SSE subscribers ─────────────────────────────────────────────────────
+
+    def subscribe(self) -> "asyncio.Queue[tuple[str, str]]":
+        """Register a new SSE subscriber.
+
+        Returns:
+            The subscriber's ``(event_name, html)`` queue.
+        """
+        queue: asyncio.Queue[tuple[str, str]] = asyncio.Queue(maxsize=_QUEUE_MAXSIZE)
+        self._subscribers.add(queue)
+        return queue
+
+    def unsubscribe(self, queue: "asyncio.Queue[tuple[str, str]]") -> None:
+        """Remove an SSE subscriber.
+
+        Args:
+            queue: The queue previously returned by :meth:`subscribe`.
+        """
+        self._subscribers.discard(queue)
+
+    def _broadcast(self, event: str, fragment: str) -> None:
+        """Fan one event out to all subscribers, dropping oldest on backlog.
+
+        Args:
+            event: SSE event name (``panel:<name>`` or ``board``).
+            fragment: The rendered HTML payload.
+        """
+        for queue in self._subscribers:
+            try:
+                queue.put_nowait((event, fragment))
+            except asyncio.QueueFull:
+                try:
+                    queue.get_nowait()
+                except asyncio.QueueEmpty:
+                    pass
+                try:
+                    queue.put_nowait((event, fragment))
+                except asyncio.QueueFull:
+                    pass