tinyjukebox 0.47.0__py3-none-any.whl

tinyjukebox/core/pubsub/base.py

@@ -0,0 +1,196 @@
+"""Common base class of all pubsub clients."""
+
+import logging
+import threading
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from typing import Any
+
+from tinyjukebox.core import unexpected
+from tinyjukebox.core.logging import get_child
+from tinyjukebox.core.pubsub.statistics import Statistics
+
+__all__ = ["MessageHandler", "NotConnectedError", "Payload", "PubSubClient", "PubSubError"]
+
+Payload = object
+"""Any (immutable) Python object - payloads are passed to the handlers unchanged (see :mod:`tinyjukebox.messages`)."""
+
+MessageHandler = Callable[[Any], object]
+"""A message handler receives the payload of a message."""
+
+
+class PubSubError(RuntimeError):
+    """Base class for all errors raised by pubsub clients."""
+
+
+class NotConnectedError(PubSubError):
+    """Raised if a message should be published while the client is not connected."""
+
+
+class PubSubClient(ABC):
+    """Topic/handler registry shared by all pubsub implementations.
+
+    Any number of handlers can be registered per topic; every handler receives the payload of each
+    message published to a matching topic (wildcards: ``+`` one level, ``#`` all remaining levels). Subscriptions
+    are independent of the connection state: they can be added before :meth:`connect` and are re-activated on every
+    (re)connect.
+
+    ``is_connected`` and ``is_disconnected`` are ``threading.Event`` instances that reflect the connection state.
+    """
+
+    def __init__(self, logger: logging.Logger, logger_name: str) -> None:
+        self._logger = get_child(logger, logger_name)
+        self._logger.info("__init__ - initializing")
+        self.is_connected = threading.Event()
+        self.is_disconnected = threading.Event()
+        self.is_disconnected.set()
+        # _lock_client serializes (dis)connect/(un)subscribe.
+        # _handlers_lock guards the registry only and is held briefly; the thread dispatching messages must never
+        # wait for _lock_client (dead-lock while e.g. disconnect() waits for the dispatch thread).
+        self._lock_client = threading.Lock()
+        self._handlers_lock = threading.Lock()
+        self._topic_handler: dict[str, list[MessageHandler]] = {}
+        self.stats = Statistics()
+
+    @property
+    def logger(self) -> logging.Logger:
+        """The logger of this client."""
+        return self._logger
+
+    @abstractmethod
+    def connect(self) -> None:
+        """Connect and activate all registered subscriptions."""
+
+    @abstractmethod
+    def disconnect(self) -> None:
+        """Disconnect; the registered subscriptions are kept."""
+
+    @abstractmethod
+    def publish(self, topic: str, msg: Payload) -> None:
+        """Publish ``msg`` to ``topic``.
+
+        :raises NotConnectedError: if the client is not connected.
+        """
+
+    def published_at(self) -> float | None:
+        """``time.monotonic()`` of the publication of the message that is currently being dispatched (to be called
+        by message handlers); ``None`` if the client does not know it."""
+        return None
+
+    @abstractmethod
+    def _activate_subscription(self, topic: str) -> None:
+        """Called (with ``_lock_client`` held) after the first handler for ``topic`` has been registered.
+
+        Implementations must not access the handler registry directly.
+        """
+
+    @abstractmethod
+    def _deactivate_subscriptions(self, topics: list[str]) -> None:
+        """Called (with ``_lock_client`` held) after the last handler of each of ``topics`` has been removed."""
+
+    def subscribe(self, topic: str, handler: MessageHandler, ignore_duplicate: bool = False) -> None:
+        """Register ``handler`` for ``topic``.
+
+        :param ignore_duplicate: if false, registering the same topic/handler pair twice raises a ``ValueError``;
+            if true, the second registration is silently ignored.
+        """
+        self._logger.info("subscribe - topic '%s', handler '%s'.", topic, _handler_name(handler))
+        with self._lock_client:
+            with self._handlers_lock:
+                handlers = self._topic_handler.get(topic)
+                if handlers is not None:
+                    if handler not in handlers:
+                        handlers.append(handler)
+                    elif not ignore_duplicate:
+                        raise ValueError(f"subscribe - topic/handler pair already added. ({topic}/{handler})")
+                    return
+                self._topic_handler[topic] = [handler]
+            self._activate_subscription(topic)
+
+    def unsubscribe(self, topic: str, handler: MessageHandler, ignore_not_found: bool = False) -> None:
+        """Remove a topic/handler pair (the reverse of :meth:`subscribe`).
+
+        Other handlers of the same topic stay registered. The topic itself is unsubscribed as soon as no handler
+        is left.
+
+        :param ignore_not_found: if false, unknown topic/handler pairs raise a ``ValueError``.
+        """
+        self._logger.info("unsubscribe - topic '%s', handler '%s'.", topic, _handler_name(handler))
+        with self._lock_client:
+            with self._handlers_lock:
+                handlers = self._topic_handler.get(topic)
+                if handlers is None or handler not in handlers:
+                    if ignore_not_found:
+                        return
+                    raise ValueError(f"unsubscribe - unknown topic/handler pair ({topic}/{handler}).")
+                handlers.remove(handler)
+                if handlers:
+                    return
+                del self._topic_handler[topic]
+            self._deactivate_subscriptions([topic])
+
+    def unsubscribe_all(self) -> None:
+        """Remove all subscriptions."""
+        with self._lock_client:
+            with self._handlers_lock:
+                topics = list(self._topic_handler)
+                self._topic_handler.clear()
+            self._logger.info("unsubscribe_all - topics %s", topics)
+            if topics:
+                self._deactivate_subscriptions(topics)
+
+    def subscribed_topics(self) -> dict[str, list[str]]:
+        """All subscribed topics with the (qualified) names of their handlers."""
+        with self._handlers_lock:
+            return {topic: [_handler_name(h) for h in handlers] for topic, handlers in self._topic_handler.items()}
+
+    def _topics(self) -> list[str]:
+        """All subscribed topics (snapshot)."""
+        with self._handlers_lock:
+            return list(self._topic_handler)
+
+    def get_stats(self) -> dict[str, Any]:
+        """Connection state, message statistics and subscriptions."""
+        return {
+            "connected": self.is_connected.is_set(),
+            "message_stats": self.stats.get_stats(),
+            "subscribed_topics": self.subscribed_topics(),
+        }
+
+    def _handlers_for(self, topic: str) -> list[MessageHandler]:
+        """All handlers whose subscription matches ``topic`` (incl. wildcard subscriptions)."""
+        with self._handlers_lock:
+            handlers = list(self._topic_handler.get(topic, ()))
+            for subscription, subscription_handlers in self._topic_handler.items():
+                if subscription != topic and _is_wildcard(subscription) and topic_matches(subscription, topic):
+                    handlers.extend(h for h in subscription_handlers if h not in handlers)
+            return handlers
+
+    def _dispatch(self, topic: str, payload: Payload) -> None:
+        """Call all handlers registered for ``topic``. Exceptions of handlers are logged and swallowed."""
+        self.stats.recv(topic)
+        for handler in self._handlers_for(topic):
+            try:
+                handler(payload)
+            except Exception as error:
+                self._logger.exception("message handler '%s' for topic '%s' failed.", _handler_name(handler), topic)
+                unexpected.report(error, f"handler {_handler_name(handler)} of the topic {topic}")
+
+
+def topic_matches(subscription: str, topic: str) -> bool:
+    """True if ``topic`` matches ``subscription`` (``+``: exactly one level, ``#``: all remaining levels, also none)."""
+    levels = topic.split("/")
+    for index, part in enumerate(subscription.split("/")):
+        if part == "#":
+            return True
+        if index >= len(levels) or part not in ("+", levels[index]):
+            return False
+    return len(subscription.split("/")) == len(levels)
+
+
+def _is_wildcard(topic: str) -> bool:
+    return "+" in topic or "#" in topic
+
+
+def _handler_name(handler: MessageHandler) -> str:
+    return getattr(handler, "__qualname__", repr(handler))

tinyjukebox/core/pubsub/localqueue.py

@@ -0,0 +1,102 @@
+"""In-process pubsub client."""
+
+import logging
+import queue
+import threading
+import time
+
+from tinyjukebox.core.pubsub.base import NotConnectedError, Payload, PubSubClient
+from tinyjukebox.core.threads import LoggerThread
+
+__all__ = ["LocalQueue"]
+
+_Item = tuple[str, Payload, float] | None  # topic, payload, time.monotonic() of the publication
+
+
+class LocalQueue(PubSubClient):
+    """Pubsub client that delivers messages within the process via a ``queue.Queue``.
+
+    Messages are delivered asynchronously and in order by a single worker thread - handlers must therefore never
+    block. Handlers receive the published object itself (no copy).
+    """
+
+    def __init__(self, logger: logging.Logger) -> None:
+        super().__init__(logger, "LocalQueue")
+        self._message_queue: queue.Queue[_Item] = queue.Queue()  # a new one with every connection
+        self._worker: LoggerThread | None = None  # the worker of the connection
+        # all running workers: a handler may disconnect - its worker then runs until the handler returns. An own
+        # lock: a worker must never wait for _lock_client
+        self._workers: set[threading.Thread] = set()
+        self._workers_lock = threading.Lock()
+        self._local = threading.local()  # published_at of the message a worker dispatches
+
+    def _work(self, messages: "queue.Queue[_Item]") -> None:
+        try:
+            while (item := messages.get()) is not None:
+                topic, payload, self._local.published_at = item
+                # nothing after a disconnect: neither while the client is disconnected nor once it has connected again
+                if messages is self._message_queue and self.is_connected.is_set():
+                    self._dispatch(topic, payload)
+                self._local.published_at = None
+                messages.task_done()
+            messages.task_done()
+        finally:
+            with self._workers_lock:
+                self._workers.discard(threading.current_thread())
+
+    def connect(self) -> None:
+        with self._lock_client:
+            if self.is_connected.is_set():
+                self._logger.info("connect - already connected.")
+                return
+            self._message_queue = queue.Queue()
+            worker = LoggerThread(
+                target=self._work, args=(self._message_queue,), name="LocalQueue", logger=self._logger, daemon=True
+            )
+            self._worker = worker
+            with self._workers_lock:
+                self._workers.add(worker)
+            worker.start()
+            self.is_disconnected.clear()
+            self.is_connected.set()
+
+    def disconnect(self) -> None:
+        with self._lock_client:
+            worker = self._worker
+            if worker is None:
+                return
+            self.is_connected.clear()
+            self._message_queue.put(None)
+            self._worker = None
+        if worker is not threading.current_thread():  # disconnect might be called by a message handler
+            worker.join()
+        self.is_disconnected.set()
+
+    def publish(self, topic: str, msg: Payload) -> None:
+        self._logger.debug("publish - topic '%s', message '%s'.", topic, msg)
+        if not self.is_connected.is_set():
+            raise NotConnectedError("publish - trying to publish while not being connected.")
+        self.stats.sent(topic)
+        self._message_queue.put((topic, msg, time.monotonic()))
+
+    def wait_until_delivered(self) -> None:
+        """Block until all published messages have been delivered to their handlers.
+
+        Returns immediately if called by a message handler - the worker would wait for itself.
+        """
+        if self._is_worker():
+            return
+        self._message_queue.join()
+
+    def published_at(self) -> float | None:
+        return getattr(self._local, "published_at", None)
+
+    def _is_worker(self) -> bool:
+        with self._workers_lock:
+            return threading.current_thread() in self._workers
+
+    def _activate_subscription(self, topic: str) -> None:
+        pass
+
+    def _deactivate_subscriptions(self, topics: list[str]) -> None:
+        pass

tinyjukebox/core/pubsub/statistics.py

@@ -0,0 +1,57 @@
+"""Message counters per topic."""
+
+import threading
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Any, NamedTuple
+
+__all__ = ["StatEntry", "Statistics", "Totals"]
+
+
+@dataclass
+class StatEntry:
+    """Counter for sent and received messages of one topic."""
+
+    sent_messages: int = 0
+    received_messages: int = 0
+
+
+class Totals(NamedTuple):
+    received_messages: int
+    sent_messages: int
+
+
+class Statistics:
+    """Thread-safe sent/received counters for all topics."""
+
+    def __init__(self) -> None:
+        self.stats: defaultdict[str, StatEntry] = defaultdict(StatEntry)
+        self._lock = threading.Lock()
+
+    def sent(self, topic: str) -> None:
+        """Increase the sent message counter of ``topic``."""
+        with self._lock:
+            self.stats[topic].sent_messages += 1
+
+    def recv(self, topic: str) -> None:
+        """Increase the received message counter of ``topic``."""
+        with self._lock:
+            self.stats[topic].received_messages += 1
+
+    def get_totals(self) -> Totals:
+        """Sum of all counters."""
+        with self._lock:
+            return Totals(
+                received_messages=sum(s.received_messages for s in self.stats.values()),
+                sent_messages=sum(s.sent_messages for s in self.stats.values()),
+            )
+
+    def get_stats(self) -> dict[str, Any]:
+        """Totals and per-topic counters as a JSON-serializable dict."""
+        received, sent = self.get_totals()
+        with self._lock:
+            topics = {
+                topic: {"messages-received": s.received_messages, "messages-sent": s.sent_messages}
+                for topic, s in self.stats.items()
+            }
+        return {"received": received, "sent": sent, "topics": topics}

tinyjukebox/core/schema/__init__.py

@@ -0,0 +1,5 @@
+"""JSON schemas of the common config sections (``logger``) and of the complete config file."""
+
+from tinyjukebox.core.schema.microservice import get_schema
+
+__all__ = ["get_schema"]

tinyjukebox/core/schema/logger.py

@@ -0,0 +1,40 @@
+"""Schema of the ``logger`` config section."""
+
+from typing import Any
+
+KEY = "logger"
+
+LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+
+
+def get_schema() -> tuple[str, dict[str, Any]]:
+    schema = {
+        "description": "Logger configuration",
+        "type": "object",
+        "properties": {
+            "log-level": {"description": "Log level to be used.", "type": "string", "enum": LOG_LEVELS},
+            "log-file": {"description": "File name for the logger.", "type": "string"},
+            "log-rotation": {
+                "description": "Parameters for log rotation (optional).",
+                "type": "object",
+                "properties": {
+                    "maxbytes": {
+                        "type": "integer",
+                        "minimum": 1,
+                        "description": "Rollover occurs whenever the current log file is nearly maxbytes in length.",
+                    },
+                    "backupcount": {
+                        "type": "integer",
+                        "minimum": 1,
+                        "description": "With a backupcount of 5 and a base file name of app.log, you would get "
+                        "app.log, app.log.1, app.log.2, up to app.log.5.",
+                    },
+                },
+                "required": ["maxbytes", "backupcount"],
+                "additionalProperties": False,
+            },
+        },
+        "required": ["log-level", "log-file"],
+        "additionalProperties": False,
+    }
+    return KEY, schema

tinyjukebox/core/schema/microservice.py

@@ -0,0 +1,48 @@
+"""Schema of a complete microservice config file."""
+
+import json
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from tinyjukebox.core.schema import logger
+
+__all__ = ["dump_schema", "get_schema"]
+
+JSON_SCHEMA_DRAFT = "http://json-schema.org/draft-07/schema#"
+
+
+def get_schema(sub_schema: Mapping[str, Any], definitions: Mapping[str, Any] | None = None) -> dict[str, Any]:
+    """Combine the service specific ``sub_schema`` with the section ``logger``.
+
+    :param sub_schema: mapping of root keys to their schemas; all of them are required.
+    :param definitions: shared schema definitions (referenced via ``#/definitions/<name>``).
+    """
+    schema: dict[str, Any] = {
+        "$schema": JSON_SCHEMA_DRAFT,
+        "title": "Configuration of the Tiny Jukebox box.",
+        "type": "object",
+        "properties": dict(sub_schema),
+        "required": list(sub_schema),
+        "definitions": dict(definitions or {}),
+    }
+    key, sub = logger.get_schema()
+    schema["properties"][key] = sub
+    schema["required"].append(key)
+    return schema
+
+
+def dump_schema(schema: Mapping[str, Any], filename: str | Path) -> bool:
+    """Write ``schema`` as JSON to ``filename`` if the content differs from the file.
+
+    :return: ``True`` if the file has been (re)written.
+    """
+    path = Path(filename)
+    new_content = json.dumps(schema, indent=4)
+    try:
+        if path.read_text(encoding="utf-8") == new_content:
+            return False
+    except OSError:
+        pass
+    path.write_text(new_content, encoding="utf-8")
+    return True

tinyjukebox/core/threads.py

@@ -0,0 +1,39 @@
+"""Threading helpers."""
+
+import logging
+import threading
+from collections.abc import Callable, Iterable, Mapping
+from typing import Any
+
+from tinyjukebox.core import unexpected
+
+__all__ = ["LoggerThread"]
+
+
+class LoggerThread(threading.Thread):
+    """``threading.Thread`` that logs uncaught exceptions of its target before re-raising them.
+
+    Exceptions in plain threads only end up on stderr. Services running as daemons should have them in the log file.
+    """
+
+    def __init__(
+        self,
+        group: None = None,
+        target: Callable[..., object] | None = None,
+        name: str | None = None,
+        args: Iterable[Any] = (),
+        kwargs: Mapping[str, Any] | None = None,
+        *,
+        daemon: bool | None = None,
+        logger: logging.Logger | None = None,
+    ) -> None:
+        super().__init__(group=group, target=target, name=name, args=args, kwargs=kwargs, daemon=daemon)
+        self._logger = logger if logger is not None else logging.getLogger(__name__)
+
+    def run(self) -> None:
+        try:
+            super().run()
+        except Exception as error:
+            self._logger.exception("uncaught exception in thread '%s'", self.name)
+            unexpected.report(error, f"thread {self.name}")
+            raise

tinyjukebox/core/ui/__init__.py

@@ -0,0 +1,5 @@
+"""Interactive command line of a running service (based on :mod:`cmd`)."""
+
+from tinyjukebox.core.ui.cli import UI, Command
+
+__all__ = ["UI", "Command"]

tinyjukebox/core/ui/cli.py

@@ -0,0 +1,120 @@
+"""Interactive command line interface running in its own thread."""
+
+import cmd
+import contextlib
+import logging
+import threading
+from collections.abc import Callable, Mapping
+
+from tinyjukebox.core.logging import get_child
+from tinyjukebox.core.threads import LoggerThread
+
+__all__ = ["UI", "Command"]
+
+Command = Callable[[str], object]
+"""A command receives the rest of the input line as argument. Its docstring is shown by ``help``."""
+
+
+class UI(cmd.Cmd):
+    """Command line of a service (``help`` lists all commands, ``exit``/``e``/``q`` stops the service).
+
+    The end of the input (``Ctrl-D``) stops the service only on a terminal - without one (e.g. ``< /dev/null``) it
+    ends only the command line.
+
+    :param intro: text shown at start.
+    :param prompt: prompt string.
+    :param stop_service: called (in a separate thread) if the user leaves the command line.
+    :param service_is_stopping: event that is set while the service is stopping (then ``stop_service`` is not
+        called again).
+    :param logger: parent logger.
+    :param commands: additional commands - name to callable.
+    """
+
+    def __init__(
+        self,
+        intro: str,
+        prompt: str,
+        stop_service: Callable[[], object],
+        service_is_stopping: threading.Event,
+        logger: logging.Logger,
+        commands: Mapping[str, Command] | None = None,
+    ) -> None:
+        super().__init__()
+        self._logger = get_child(logger, type(self).__name__)
+        self.intro = intro
+        self.prompt = prompt
+        self._stop_service = stop_service
+        self._service_is_stopping = service_is_stopping
+        self._closed = False  # stopped by the service: a line typed afterwards is not executed
+        self._end_of_input = False
+        self._thread = LoggerThread(target=self._loop, name="UI", logger=self._logger, daemon=True)
+        for name, function in (commands or {}).items():
+            self.add_command(name, function)
+
+    def add_command(self, name: str, function: Command) -> None:
+        """Add the command ``name``.
+
+        :raises ValueError: if a command with this name exists already.
+        :raises TypeError: if ``function`` is not callable.
+        """
+        if hasattr(self, "do_" + name):
+            raise ValueError(f"command '{name}' already defined")
+        if not callable(function):
+            raise TypeError("parameter function must be callable")
+        setattr(self, "do_" + name, function)
+
+    def get_names(self) -> list[str]:
+        # cmd.Cmd only inspects the class - include the commands added to this instance
+        return dir(self)
+
+    def start(self) -> None:
+        self._thread.start()
+
+    def stop(self) -> None:
+        """End the command line (called by the service when it stops)."""
+        self._closed = True
+        if self._thread.is_alive() and self._thread is not threading.current_thread():
+            # cmdloop blocks in input() until the next line; the thread is a daemon and must not block the shutdown
+            # of the service
+            self._thread.join(timeout=0.5)
+
+    def _loop(self) -> None:
+        with contextlib.suppress(EOFError):
+            self.cmdloop()
+        if self._closed or self._service_is_stopping.is_set():
+            return
+        if self._end_of_input and not self._interactive():
+            self._logger.warning("the input has ended and is no terminal - the console ends, the service goes on")
+            return
+        LoggerThread(target=self._stop_service, name="UI.stop_service", logger=self._logger, daemon=True).start()
+
+    def _interactive(self) -> bool:
+        try:
+            return bool(self.stdin.isatty())
+        except (AttributeError, OSError, ValueError):  # no stdin, or a closed one
+            return False
+
+    def onecmd(self, line: str) -> bool:
+        if self._closed:
+            return True  # the service has stopped the command line meanwhile
+        try:
+            return bool(super().onecmd(line))
+        except Exception as e:
+            self._logger.exception("command '%s' failed", line)
+            print(f"Caught exception: {e}")
+            return False
+
+    def emptyline(self) -> bool:
+        return False  # do not repeat the last command
+
+    def do_exit(self, arg: str) -> bool:
+        """exit - stop the service: EXIT"""
+        return True
+
+    def do_EOF(self, arg: str) -> bool:  # noqa: N802 - name defined by cmd.Cmd
+        """Ctrl-D - stop the service"""
+        self._end_of_input = True
+        return True
+
+    do_e = do_exit
+    do_q = do_exit