hannah-logging 0.1.0__tar.gz

hannah_logging-0.1.0/PKG-INFO

@@ -0,0 +1,86 @@
+Metadata-Version: 2.4
+Name: hannah-logging
+Version: 0.1.0
+Summary: Ships the logs of a Hannah component to the Hannah log collector
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: grpcio>=1.84.0
+Requires-Dist: hannah-proto>=4.5.0
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == "test"
+
+# hannah-logging (Python)
+
+Ships the logs of a Hannah component to the Hannah log collector, so a user can download
+one archive of all components' logs from the WebUI.
+
+The library adds one `logging.Handler`. Your existing handlers (stdout, journald, syslog)
+keep working unchanged.
+
+## Installation
+
+```sh
+pip install hannah-logging
+```
+
+The library relies on the `hannah-proto` version the component already uses, because
+every call to Hannah Core has to carry the matching protocol version.
+
+## Usage
+
+```python
+import logging
+import hannah_logging
+
+logging.basicConfig(level=logging.INFO)
+
+# As early as possible: everything logged from here on is buffered.
+shipping = hannah_logging.install(
+    "telegram",
+    version=__version__,
+    secrets=[config.bot_token],                         # masked wherever they appear
+    logger_categories={"hannah.stt": hannah_logging.TRANSCRIPT},
+)
+
+# Once the config is loaded:
+shipping.connect(
+    hannah_address="localhost:50051",                   # discovery via Hannah Core
+    collector_address=None,                             # optional static fallback
+)
+```
+
+### Categories
+
+Each line is tagged `GENERAL`, `TRANSCRIPT` (speech transcripts, user utterances) or
+`METADATA` (room and device names, presence). Exports can leave out the last two, and
+the WebUI does so by default.
+
+- By logger name: `logger_categories={"hannah.stt": TRANSCRIPT}` also covers child
+  loggers such as `hannah.stt.whisper`.
+- Per call: `log.info("heard: %s", text, extra={hannah_logging.CATEGORY_ATTR: hannah_logging.TRANSCRIPT})`
+
+### Secrets
+
+Before a line leaves the process, the library masks:
+
+- values of keys such as `password`, `token`, `api_key`, `secret`, `psk`
+- `Bearer` tokens, JWTs, credentials in URLs (`mqtt://user:pw@host`), PEM private keys
+- Telegram bot tokens and well-known token prefixes (`glpat-`, `ghp_`, …)
+- every value passed as `secrets=[…]` or later through `shipping.add_secret(…)`
+
+Extra regexes go in `secret_patterns=[…]`. The component's own handlers still see the
+unmasked line.
+
+## Behaviour
+
+- **Buffer:** 4 MiB by default (`max_buffer_bytes`). When it is full, the oldest lines are
+  dropped and reported to the collector as a gap. While no collector is known, the
+  buffer simply keeps running as a ring.
+- **Timestamps** are taken when a line is logged, not when it is sent.
+- **Discovery:** subscribes to Hannah Core's infrastructure announcements and follows the
+  log collector when it moves. If Core is unreachable, the last known collector is kept.
+  The static `collector_address` is used while none is announced.
+- **Never blocks the component:** sending runs on its own threads. Connection problems
+  are logged once to the `hannah_logging` logger (not shipped) and retried with backoff.
+- **Shutdown:** at exit, the library tries for up to 2 seconds to send what is still
+  buffered (`shipping.close(timeout=…)`).

hannah_logging-0.1.0/README.md

@@ -0,0 +1,75 @@
+# hannah-logging (Python)
+
+Ships the logs of a Hannah component to the Hannah log collector, so a user can download
+one archive of all components' logs from the WebUI.
+
+The library adds one `logging.Handler`. Your existing handlers (stdout, journald, syslog)
+keep working unchanged.
+
+## Installation
+
+```sh
+pip install hannah-logging
+```
+
+The library relies on the `hannah-proto` version the component already uses, because
+every call to Hannah Core has to carry the matching protocol version.
+
+## Usage
+
+```python
+import logging
+import hannah_logging
+
+logging.basicConfig(level=logging.INFO)
+
+# As early as possible: everything logged from here on is buffered.
+shipping = hannah_logging.install(
+    "telegram",
+    version=__version__,
+    secrets=[config.bot_token],                         # masked wherever they appear
+    logger_categories={"hannah.stt": hannah_logging.TRANSCRIPT},
+)
+
+# Once the config is loaded:
+shipping.connect(
+    hannah_address="localhost:50051",                   # discovery via Hannah Core
+    collector_address=None,                             # optional static fallback
+)
+```
+
+### Categories
+
+Each line is tagged `GENERAL`, `TRANSCRIPT` (speech transcripts, user utterances) or
+`METADATA` (room and device names, presence). Exports can leave out the last two, and
+the WebUI does so by default.
+
+- By logger name: `logger_categories={"hannah.stt": TRANSCRIPT}` also covers child
+  loggers such as `hannah.stt.whisper`.
+- Per call: `log.info("heard: %s", text, extra={hannah_logging.CATEGORY_ATTR: hannah_logging.TRANSCRIPT})`
+
+### Secrets
+
+Before a line leaves the process, the library masks:
+
+- values of keys such as `password`, `token`, `api_key`, `secret`, `psk`
+- `Bearer` tokens, JWTs, credentials in URLs (`mqtt://user:pw@host`), PEM private keys
+- Telegram bot tokens and well-known token prefixes (`glpat-`, `ghp_`, …)
+- every value passed as `secrets=[…]` or later through `shipping.add_secret(…)`
+
+Extra regexes go in `secret_patterns=[…]`. The component's own handlers still see the
+unmasked line.
+
+## Behaviour
+
+- **Buffer:** 4 MiB by default (`max_buffer_bytes`). When it is full, the oldest lines are
+  dropped and reported to the collector as a gap. While no collector is known, the
+  buffer simply keeps running as a ring.
+- **Timestamps** are taken when a line is logged, not when it is sent.
+- **Discovery:** subscribes to Hannah Core's infrastructure announcements and follows the
+  log collector when it moves. If Core is unreachable, the last known collector is kept.
+  The static `collector_address` is used while none is announced.
+- **Never blocks the component:** sending runs on its own threads. Connection problems
+  are logged once to the `hannah_logging` logger (not shipped) and retried with backoff.
+- **Shutdown:** at exit, the library tries for up to 2 seconds to send what is still
+  buffered (`shipping.close(timeout=…)`).

hannah_logging-0.1.0/hannah_logging/__init__.py

@@ -0,0 +1,133 @@
+"""Ships the logs of a Hannah component to the Hannah log collector.
+
+    import hannah_logging
+
+    shipping = hannah_logging.install("core", version=__version__)   # as early as possible
+    ...
+    shipping.connect(hannah_address="localhost:50051")                # once the config is loaded
+
+Everything logged from `install` on is buffered in memory (bounded), and sent to the
+collector as soon as one is known — announced by Hannah Core, or a static address.
+The host's own handlers (stdout, journald, syslog) are not touched.
+"""
+from __future__ import annotations
+
+import atexit
+import logging
+import socket
+import threading
+from typing import Iterable, Mapping, Optional
+
+from hannah_proto import logging_pb2
+
+from .buffer import LogBuffer
+from .handler import CATEGORY_ATTR, GENERAL, METADATA, TRANSCRIPT, CollectorHandler
+from .secrets import SecretFilter
+from .shipper import Discovery, Shipper
+
+__all__ = [
+    "install", "LogShipping", "CollectorHandler", "SecretFilter",
+    "GENERAL", "TRANSCRIPT", "METADATA", "CATEGORY_ATTR",
+]
+
+DEFAULT_MAX_BUFFER_BYTES = 4 * 1024 * 1024
+
+
+class LogShipping:
+    """Handle returned by `install`. Owns the handler and the background threads."""
+
+    def __init__(
+        self,
+        component: str,
+        version: str,
+        instance: str,
+        handler: CollectorHandler,
+        buffer: LogBuffer,
+        secret_filter: SecretFilter,
+        target_logger: logging.Logger,
+    ):
+        self.handler = handler
+        self._hello = logging_pb2.ShipHello(component=component, instance=instance, version=version)
+        self._buffer = buffer
+        self._secret_filter = secret_filter
+        self._target_logger = target_logger
+        self._shipper: Optional[Shipper] = None
+        self._discovery: Optional[Discovery] = None
+        self._lock = threading.Lock()
+
+    def connect(self, hannah_address: Optional[str] = None, collector_address: Optional[str] = None) -> None:
+        """Starts shipping. `hannah_address` enables discovery via Hannah Core;
+        `collector_address` is the fallback while none is announced. Only the
+        first call has an effect."""
+        with self._lock:
+            if self._shipper is not None:
+                return
+            self._shipper = Shipper(self._buffer, self._hello, collector_address or None)
+            self._shipper.start()
+            if hannah_address:
+                self._discovery = Discovery(hannah_address, self._shipper.set_discovered)
+                self._discovery.start()
+
+    def add_secret(self, secret: str) -> None:
+        """Masks this literal value in everything logged from now on (e.g. a token
+        that only becomes known after `install`)."""
+        self._secret_filter.add_literal(secret)
+
+    def close(self, timeout: float = 2.0) -> None:
+        """Detaches the handler and gives the shipper up to `timeout` seconds to
+        send what is still buffered."""
+        self._target_logger.removeHandler(self.handler)
+        with self._lock:
+            discovery, shipper = self._discovery, self._shipper
+            self._discovery = self._shipper = None
+        if discovery is not None:
+            discovery.stop()
+        if shipper is not None:
+            shipper.stop()
+            shipper.join(timeout)
+
+
+def install(
+    component: str,
+    *,
+    version: str = "",
+    instance: Optional[str] = None,
+    hannah_address: Optional[str] = None,
+    collector_address: Optional[str] = None,
+    secrets: Iterable[str] = (),
+    secret_patterns: Iterable[str] = (),
+    logger_categories: Optional[Mapping[str, int]] = None,
+    level: int = logging.NOTSET,
+    max_buffer_bytes: int = DEFAULT_MAX_BUFFER_BYTES,
+    logger: Optional[logging.Logger] = None,
+) -> LogShipping:
+    """Attaches a CollectorHandler to `logger` (default: the root logger) and starts
+    buffering. Pass `hannah_address`/`collector_address` here or later via
+    `LogShipping.connect`.
+
+    component          e.g. "core", "telegram" — the name shown in the export
+    instance           distinguishes several instances; defaults to the hostname
+    secrets            literal values to mask (the component's own tokens/passwords)
+    secret_patterns    extra regexes to mask, on top of the built-in ones
+    logger_categories  logger name prefix -> category, e.g. {"hannah.stt": TRANSCRIPT};
+                       a single call can override it with extra={CATEGORY_ATTR: ...}
+    """
+    buffer = LogBuffer(max_buffer_bytes)
+    secret_filter = SecretFilter(secrets, secret_patterns)
+    handler = CollectorHandler(buffer, secret_filter, logger_categories, level)
+    target_logger = logger or logging.getLogger()
+    target_logger.addHandler(handler)
+
+    shipping = LogShipping(
+        component=component,
+        version=version,
+        instance=instance or socket.gethostname(),
+        handler=handler,
+        buffer=buffer,
+        secret_filter=secret_filter,
+        target_logger=target_logger,
+    )
+    atexit.register(shipping.close)
+    if hannah_address or collector_address:
+        shipping.connect(hannah_address, collector_address)
+    return shipping

hannah_logging-0.1.0/hannah_logging/buffer.py

@@ -0,0 +1,84 @@
+"""Bounded in-memory buffer between the logging handler and the shipper thread."""
+from __future__ import annotations
+
+import collections
+import threading
+from dataclasses import dataclass
+from typing import Deque, List, Optional, Tuple
+
+# Rough per-entry overhead on top of message + logger name, so a flood of empty
+# messages still counts against the limit.
+_ENTRY_OVERHEAD = 64
+
+
+@dataclass(frozen=True)
+class Entry:
+    timestamp_ms: int
+    level: int      # hannah_proto LogLevel value
+    logger: str
+    message: str
+    category: int   # hannah_proto LogCategory value
+
+    def size(self) -> int:
+        return len(self.message) + len(self.logger) + _ENTRY_OVERHEAD
+
+
+@dataclass
+class Gap:
+    dropped: int
+    from_ms: int
+    to_ms: int
+
+
+class LogBuffer:
+    """Ring buffer limited by size. When full, the oldest entries are dropped and
+    recorded as a gap, which is shipped before the remaining entries.
+
+    `put` never blocks the logging thread beyond a short lock."""
+
+    def __init__(self, max_bytes: int):
+        self._max_bytes = max_bytes
+        self._entries: Deque[Entry] = collections.deque()
+        self._bytes = 0
+        self._gap: Optional[Gap] = None
+        self._cond = threading.Condition()
+
+    def put(self, entry: Entry) -> None:
+        with self._cond:
+            self._entries.append(entry)
+            self._bytes += entry.size()
+            while self._bytes > self._max_bytes and self._entries:
+                self._drop_oldest()
+            self._cond.notify()
+
+    def _drop_oldest(self) -> None:
+        old = self._entries.popleft()
+        self._bytes -= old.size()
+        if self._gap is None:
+            self._gap = Gap(dropped=1, from_ms=old.timestamp_ms, to_ms=old.timestamp_ms)
+        else:
+            self._gap.dropped += 1
+            self._gap.to_ms = max(self._gap.to_ms, old.timestamp_ms)
+
+    def take(self, max_items: int, timeout: Optional[float]) -> Tuple[Optional[Gap], List[Entry]]:
+        """Waits up to `timeout` for data, then removes and returns the pending gap
+        (if any) and up to `max_items` entries, oldest first."""
+        with self._cond:
+            if not self._entries and self._gap is None:
+                self._cond.wait(timeout)
+            gap, self._gap = self._gap, None
+            items: List[Entry] = []
+            while self._entries and len(items) < max_items:
+                entry = self._entries.popleft()
+                self._bytes -= entry.size()
+                items.append(entry)
+            return gap, items
+
+    def wake(self) -> None:
+        """Wakes a waiting `take`, e.g. to react to a new collector address or shutdown."""
+        with self._cond:
+            self._cond.notify_all()
+
+    def __len__(self) -> int:
+        with self._cond:
+            return len(self._entries)

hannah_logging-0.1.0/hannah_logging/handler.py

@@ -0,0 +1,75 @@
+"""logging.Handler that feeds the buffer. Runs in addition to the host's own handlers."""
+from __future__ import annotations
+
+import logging
+from typing import Mapping, Optional
+
+from hannah_proto import logging_pb2
+
+from .buffer import Entry, LogBuffer
+from .secrets import SecretFilter
+from .shipper import OWN_THREADS
+
+GENERAL = logging_pb2.LOG_CATEGORY_GENERAL
+TRANSCRIPT = logging_pb2.LOG_CATEGORY_TRANSCRIPT
+METADATA = logging_pb2.LOG_CATEGORY_METADATA
+
+# Name of the `extra=` key that tags a single log call, e.g.
+# log.info("heard: %s", text, extra={CATEGORY_ATTR: TRANSCRIPT})
+CATEGORY_ATTR = "log_category"
+
+
+def to_log_level(levelno: int) -> int:
+    if levelno >= logging.CRITICAL:
+        return logging_pb2.LOG_LEVEL_CRITICAL
+    if levelno >= logging.ERROR:
+        return logging_pb2.LOG_LEVEL_ERROR
+    if levelno >= logging.WARNING:
+        return logging_pb2.LOG_LEVEL_WARNING
+    if levelno >= logging.INFO:
+        return logging_pb2.LOG_LEVEL_INFO
+    return logging_pb2.LOG_LEVEL_DEBUG
+
+
+class CollectorHandler(logging.Handler):
+    """Formats, filters and buffers each record. The timestamp is the record's
+    creation time, so entries buffered before the collector was reachable keep
+    their original time."""
+
+    def __init__(
+        self,
+        buffer: LogBuffer,
+        secret_filter: SecretFilter,
+        logger_categories: Optional[Mapping[str, int]] = None,
+        level: int = logging.NOTSET,
+    ):
+        super().__init__(level)
+        self._buffer = buffer
+        self._secret_filter = secret_filter
+        # Longest prefix first, so "hannah.stt.transcript" wins over "hannah.stt".
+        self._logger_categories = sorted((logger_categories or {}).items(), key=lambda kv: len(kv[0]), reverse=True)
+        self.setFormatter(logging.Formatter("%(message)s"))
+
+    def emit(self, record: logging.LogRecord) -> None:
+        if record.thread in OWN_THREADS:
+            return
+        try:
+            message = self._secret_filter(self.format(record))
+            self._buffer.put(Entry(
+                timestamp_ms=int(record.created * 1000),
+                level=to_log_level(record.levelno),
+                logger=record.name,
+                message=message,
+                category=self._category(record),
+            ))
+        except Exception:
+            self.handleError(record)
+
+    def _category(self, record: logging.LogRecord) -> int:
+        explicit = getattr(record, CATEGORY_ATTR, None)
+        if explicit:
+            return int(explicit)
+        for prefix, category in self._logger_categories:
+            if record.name == prefix or record.name.startswith(prefix + "."):
+                return category
+        return GENERAL

hannah_logging-0.1.0/hannah_logging/secrets.py

@@ -0,0 +1,69 @@
+"""Masks secrets in a log message before it leaves the process."""
+from __future__ import annotations
+
+import re
+from typing import Iterable, List, Pattern, Tuple
+
+MASK = "***"
+
+# Key names whose value is treated as secret in "key=value", "key: value" and JSON-ish "key": "value".
+_SECRET_KEYS = (
+    r"password|passwd|pwd|passphrase|secret|client[_-]?secret|token|access[_-]?token|refresh[_-]?token"
+    r"|api[_-]?key|apikey|access[_-]?key|private[_-]?key|psk|auth|authorization|credentials?"
+)
+
+_BUILTIN_PATTERNS: List[Tuple[Pattern[str], str]] = [
+    # PEM blocks (private keys, certificates with keys).
+    (re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----", re.S), MASK),
+    # "Authorization: Bearer abc", "Bearer abc"
+    (re.compile(r"(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]+"), r"\1 " + MASK),
+    # key=value / key: value / "key": "value" — the key has to *end* in a secret word,
+    # so "bot_token=…" is masked but "tokens=512" or "token_count=3" is not.
+    (
+        re.compile(
+            r"""(?ix)
+            (?P<key>["']?[\w.-]*(?:""" + _SECRET_KEYS + r""")["']?)
+            (?P<sep>\s*[:=]\s*)
+            (?P<value>"[^"]*"|'[^']*'|[^\s,;&)}\]]+)
+            """
+        ),
+        lambda m: m.group("key") + m.group("sep") + _quoted_mask(m.group("value")),
+    ),
+    # Credentials in URLs: scheme://user:password@host
+    (re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://[^/\s:@]+):[^/\s@]+@"), r"\1:" + MASK + "@"),
+    # JWTs
+    (re.compile(r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+"), MASK),
+    # Telegram bot tokens
+    (re.compile(r"\b\d{8,10}:[A-Za-z0-9_-]{35}\b"), MASK),
+    # Well-known token prefixes (GitLab, GitHub, npm, PyPI, Slack, OpenAI-style)
+    (re.compile(r"\b(?:glpat|gldt|glrt|ghp|gho|ghu|ghs|ghr|github_pat|npm|pypi|xox[abpr]|sk)[-_][A-Za-z0-9_-]{16,}"), MASK),
+]
+
+
+def _quoted_mask(value: str) -> str:
+    if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
+        return value[0] + MASK + value[0]
+    return MASK
+
+
+class SecretFilter:
+    """Masks well-known secret shapes plus any literal values the component knows are secret
+    (its own tokens and passwords from its config — the most reliable way to catch them)."""
+
+    def __init__(self, literals: Iterable[str] = (), patterns: Iterable[str] = ()):
+        # Longest first, so a secret that contains another one is masked as a whole.
+        self._literals = sorted({s for s in literals if s and len(s) >= 4}, key=len, reverse=True)
+        self._patterns = list(_BUILTIN_PATTERNS) + [(re.compile(p), MASK) for p in patterns]
+
+    def add_literal(self, secret: str) -> None:
+        if secret and len(secret) >= 4 and secret not in self._literals:
+            self._literals.append(secret)
+            self._literals.sort(key=len, reverse=True)
+
+    def __call__(self, message: str) -> str:
+        for literal in self._literals:
+            if literal in message:
+                message = message.replace(literal, MASK)
+        for pattern, replacement in self._patterns:
+            message = pattern.sub(replacement, message)
+        return message

hannah_logging-0.1.0/hannah_logging/shipper.py

@@ -0,0 +1,245 @@
+"""Background threads: collector discovery via Hannah Core, and shipping to the collector.
+
+Both run on their own threads with a synchronous gRPC channel, so they never touch the
+host's event loop and never block its logging calls. Errors are swallowed and reported
+once per state change on the `hannah_logging` logger — records from these threads are
+not shipped (see CollectorHandler), so they only reach the host's other handlers.
+"""
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Callable, Dict, Iterator, Optional, Set, Tuple
+
+import grpc
+from hannah_proto import PROTO_VERSION, hannah_pb2, hannah_pb2_grpc, infrastructure_pb2
+from hannah_proto import logging_pb2, logging_pb2_grpc
+from hannah_proto.interceptor.compat_interceptor import client_compat_version_metadata
+
+from .buffer import LogBuffer
+
+_log = logging.getLogger("hannah_logging")
+
+PROTO_VERSION_METADATA_KEY = "x-proto-version"
+MAX_BACKOFF_S = 30.0
+CONNECT_TIMEOUT_S = 5.0
+BATCH_SIZE = 200
+
+# Idents of this library's threads — their log records are never shipped, so a failure
+# while shipping can't feed itself back into the buffer.
+OWN_THREADS: Set[int] = set()
+
+_HANNAH_SERVICE = hannah_pb2.DESCRIPTOR.services_by_name["HannahService"]
+_LOG_SERVICE = logging_pb2.DESCRIPTOR.services_by_name["LogService"]
+
+
+def _metadata(service, method: str) -> Tuple[Tuple[str, str], ...]:
+    return (
+        (PROTO_VERSION_METADATA_KEY, str(PROTO_VERSION)),
+        client_compat_version_metadata(service, method),
+    )
+
+
+def format_address(host: str, port: int) -> str:
+    if ":" in host and not host.startswith("["):
+        host = f"[{host}]"  # IPv6
+    return f"{host}:{port}"
+
+
+class _Worker(threading.Thread):
+    def __init__(self, name: str):
+        super().__init__(name=name, daemon=True)
+        self._stop_event = threading.Event()
+
+    def run(self) -> None:
+        OWN_THREADS.add(threading.get_ident())
+        try:
+            self._run()
+        finally:
+            OWN_THREADS.discard(threading.get_ident())
+
+    def _run(self) -> None:
+        raise NotImplementedError
+
+    @property
+    def stopping(self) -> bool:
+        return self._stop_event.is_set()
+
+
+class Discovery(_Worker):
+    """Subscribes to Hannah Core's infrastructure broadcast and reports the current
+    log collector address (or None) to `on_change`.
+
+    If Core becomes unreachable, the last known collector is kept — the collector
+    doesn't depend on Core, so logs keep flowing. The snapshot after a reconnect
+    replaces the known state completely."""
+
+    def __init__(self, hannah_address: str, on_change: Callable[[Optional[str]], None]):
+        super().__init__("hannah-logging-discovery")
+        self._hannah_address = hannah_address
+        self._on_change = on_change
+        self._collectors: Dict[str, str] = {}  # instance -> address
+        self._call = None
+        self._lock = threading.Lock()
+
+    def stop(self) -> None:
+        self._stop_event.set()
+        with self._lock:
+            if self._call is not None:
+                self._call.cancel()
+
+    def _run(self) -> None:
+        backoff = 1.0
+        reported_failure = False
+        while not self.stopping:
+            channel = grpc.insecure_channel(self._hannah_address)
+            try:
+                stub = hannah_pb2_grpc.HannahServiceStub(channel)
+                call = stub.SubscribeInfrastructure(
+                    infrastructure_pb2.InfrastructureFilter(kinds=[infrastructure_pb2.SERVICE_KIND_LOG_COLLECTOR]),
+                    metadata=_metadata(_HANNAH_SERVICE, "SubscribeInfrastructure"),
+                )
+                with self._lock:
+                    self._call = call
+                if self.stopping:
+                    call.cancel()
+                for message in call:
+                    if reported_failure:
+                        _log.info("Log collector discovery: connected to Hannah at %s", self._hannah_address)
+                        reported_failure = False
+                    backoff = 1.0
+                    self._handle(message)
+            except grpc.RpcError as exc:
+                if not self.stopping and not reported_failure:
+                    _log.warning("Log collector discovery: Hannah at %s unreachable (%s), retrying",
+                                 self._hannah_address, _rpc_error_text(exc))
+                    reported_failure = True
+            except Exception as exc:  # never let the thread die
+                if not self.stopping:
+                    _log.warning("Log collector discovery failed: %r", exc)
+                    reported_failure = True
+            finally:
+                with self._lock:
+                    self._call = None
+                channel.close()
+            self._stop_event.wait(backoff)
+            backoff = min(backoff * 2, MAX_BACKOFF_S)
+
+    def _handle(self, message: "infrastructure_pb2.InfrastructureMessage") -> None:
+        which = message.WhichOneof("payload")
+        if which == "snapshot":
+            self._collectors = {
+                s.instance: format_address(s.host, s.port)
+                for s in message.snapshot.services
+                if s.kind == infrastructure_pb2.SERVICE_KIND_LOG_COLLECTOR
+            }
+        elif which == "available":
+            s = message.available.service
+            if s.kind != infrastructure_pb2.SERVICE_KIND_LOG_COLLECTOR:
+                return
+            self._collectors[s.instance] = format_address(s.host, s.port)
+        elif which == "unavailable":
+            if message.unavailable.kind != infrastructure_pb2.SERVICE_KIND_LOG_COLLECTOR:
+                return
+            self._collectors.pop(message.unavailable.instance, None)
+        else:
+            return
+        # Several collectors are unusual; pick one deterministically.
+        current = self._collectors[min(self._collectors)] if self._collectors else None
+        self._on_change(current)
+
+
+class Shipper(_Worker):
+    """Streams the buffer to the collector over LogService.Ship.
+
+    Only opens the stream once the channel is ready, so nothing is taken out of the
+    buffer while the collector is unreachable. Ends the stream when the collector
+    address changes and reconnects to the new one."""
+
+    def __init__(self, buffer: LogBuffer, hello: "logging_pb2.ShipHello", static_address: Optional[str]):
+        super().__init__("hannah-logging-shipper")
+        self._buffer = buffer
+        self._hello = hello
+        self._static_address = static_address
+        self._discovered: Optional[str] = None
+        self._lock = threading.Lock()
+
+    def set_discovered(self, address: Optional[str]) -> None:
+        with self._lock:
+            changed = address != self._discovered
+            self._discovered = address
+        if changed:
+            if address:
+                _log.info("Log collector discovered at %s", address)
+            self._buffer.wake()
+
+    def target(self) -> Optional[str]:
+        with self._lock:
+            return self._discovered or self._static_address
+
+    def stop(self) -> None:
+        self._stop_event.set()
+        self._buffer.wake()
+
+    def _run(self) -> None:
+        backoff = 1.0
+        reported_failure = False
+        while not self.stopping:
+            target = self.target()
+            if not target:
+                # No collector known: the buffer keeps running as a ring, nothing to do.
+                self._stop_event.wait(1.0)
+                continue
+            channel = grpc.insecure_channel(target)
+            try:
+                grpc.channel_ready_future(channel).result(timeout=CONNECT_TIMEOUT_S)
+                if reported_failure:
+                    _log.info("Log collector at %s reachable again", target)
+                    reported_failure = False
+                logging_pb2_grpc.LogServiceStub(channel).Ship(
+                    self._messages(target),
+                    metadata=_metadata(_LOG_SERVICE, "Ship"),
+                )
+                backoff = 1.0
+                continue
+            except grpc.FutureTimeoutError:
+                failure = "not reachable"
+            except grpc.RpcError as exc:
+                failure = _rpc_error_text(exc)
+            except Exception as exc:  # never let the thread die
+                failure = repr(exc)
+            finally:
+                channel.close()
+            if not self.stopping and not reported_failure:
+                _log.warning("Log collector at %s: %s — buffering, retrying", target, failure)
+                reported_failure = True
+            self._stop_event.wait(backoff)
+            backoff = min(backoff * 2, MAX_BACKOFF_S)
+
+    def _messages(self, target: str) -> Iterator["logging_pb2.ShipMessage"]:
+        yield logging_pb2.ShipMessage(hello=self._hello)
+        while True:
+            gap, entries = self._buffer.take(BATCH_SIZE, timeout=1.0)
+            if gap is not None:
+                yield logging_pb2.ShipMessage(gap=logging_pb2.LogGap(
+                    dropped=gap.dropped, from_ms=gap.from_ms, to_ms=gap.to_ms,
+                ))
+            for e in entries:
+                yield logging_pb2.ShipMessage(entry=logging_pb2.LogEntry(
+                    timestamp_ms=e.timestamp_ms, level=e.level, logger=e.logger,
+                    message=e.message, category=e.category,
+                ))
+            # Collector moved: what's left in the buffer goes to the new one.
+            if self.target() != target:
+                return
+            # Shutdown: close the stream only once the buffer is drained.
+            if self.stopping and not entries and gap is None:
+                return
+
+
+def _rpc_error_text(exc: grpc.RpcError) -> str:
+    code = getattr(exc, "code", None)
+    details = getattr(exc, "details", None)
+    if callable(code) and callable(details):
+        return f"{code().name}: {details()}"
+    return repr(exc)

hannah_logging-0.1.0/hannah_logging.egg-info/PKG-INFO

@@ -0,0 +1,86 @@
+Metadata-Version: 2.4
+Name: hannah-logging
+Version: 0.1.0
+Summary: Ships the logs of a Hannah component to the Hannah log collector
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: grpcio>=1.84.0
+Requires-Dist: hannah-proto>=4.5.0
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == "test"
+
+# hannah-logging (Python)
+
+Ships the logs of a Hannah component to the Hannah log collector, so a user can download
+one archive of all components' logs from the WebUI.
+
+The library adds one `logging.Handler`. Your existing handlers (stdout, journald, syslog)
+keep working unchanged.
+
+## Installation
+
+```sh
+pip install hannah-logging
+```
+
+The library relies on the `hannah-proto` version the component already uses, because
+every call to Hannah Core has to carry the matching protocol version.
+
+## Usage
+
+```python
+import logging
+import hannah_logging
+
+logging.basicConfig(level=logging.INFO)
+
+# As early as possible: everything logged from here on is buffered.
+shipping = hannah_logging.install(
+    "telegram",
+    version=__version__,
+    secrets=[config.bot_token],                         # masked wherever they appear
+    logger_categories={"hannah.stt": hannah_logging.TRANSCRIPT},
+)
+
+# Once the config is loaded:
+shipping.connect(
+    hannah_address="localhost:50051",                   # discovery via Hannah Core
+    collector_address=None,                             # optional static fallback
+)
+```
+
+### Categories
+
+Each line is tagged `GENERAL`, `TRANSCRIPT` (speech transcripts, user utterances) or
+`METADATA` (room and device names, presence). Exports can leave out the last two, and
+the WebUI does so by default.
+
+- By logger name: `logger_categories={"hannah.stt": TRANSCRIPT}` also covers child
+  loggers such as `hannah.stt.whisper`.
+- Per call: `log.info("heard: %s", text, extra={hannah_logging.CATEGORY_ATTR: hannah_logging.TRANSCRIPT})`
+
+### Secrets
+
+Before a line leaves the process, the library masks:
+
+- values of keys such as `password`, `token`, `api_key`, `secret`, `psk`
+- `Bearer` tokens, JWTs, credentials in URLs (`mqtt://user:pw@host`), PEM private keys
+- Telegram bot tokens and well-known token prefixes (`glpat-`, `ghp_`, …)
+- every value passed as `secrets=[…]` or later through `shipping.add_secret(…)`
+
+Extra regexes go in `secret_patterns=[…]`. The component's own handlers still see the
+unmasked line.
+
+## Behaviour
+
+- **Buffer:** 4 MiB by default (`max_buffer_bytes`). When it is full, the oldest lines are
+  dropped and reported to the collector as a gap. While no collector is known, the
+  buffer simply keeps running as a ring.
+- **Timestamps** are taken when a line is logged, not when it is sent.
+- **Discovery:** subscribes to Hannah Core's infrastructure announcements and follows the
+  log collector when it moves. If Core is unreachable, the last known collector is kept.
+  The static `collector_address` is used while none is announced.
+- **Never blocks the component:** sending runs on its own threads. Connection problems
+  are logged once to the `hannah_logging` logger (not shipped) and retried with backoff.
+- **Shutdown:** at exit, the library tries for up to 2 seconds to send what is still
+  buffered (`shipping.close(timeout=…)`).

hannah_logging-0.1.0/hannah_logging.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+README.md
+pyproject.toml
+hannah_logging/__init__.py
+hannah_logging/buffer.py
+hannah_logging/handler.py
+hannah_logging/secrets.py
+hannah_logging/shipper.py
+hannah_logging.egg-info/PKG-INFO
+hannah_logging.egg-info/SOURCES.txt
+hannah_logging.egg-info/dependency_links.txt
+hannah_logging.egg-info/requires.txt
+hannah_logging.egg-info/top_level.txt
+tests/test_buffer.py
+tests/test_handler.py
+tests/test_integration.py
+tests/test_secrets.py

hannah_logging-0.1.0/hannah_logging.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

hannah_logging-0.1.0/hannah_logging.egg-info/requires.txt

@@ -0,0 +1,5 @@
+grpcio>=1.84.0
+hannah-proto>=4.5.0
+
+[test]
+pytest>=8

hannah_logging-0.1.0/hannah_logging.egg-info/top_level.txt

@@ -0,0 +1 @@
+hannah_logging

hannah_logging-0.1.0/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hannah-logging"
+version = "0.1.0"
+description = "Ships the logs of a Hannah component to the Hannah log collector"
+readme = "README.md"
+requires-python = ">=3.9"
+dependencies = [
+    "grpcio>=1.84.0",
+    # First release with LogService (logging.proto) and SubscribeInfrastructure.
+    "hannah-proto>=4.5.0",
+]
+
+[project.optional-dependencies]
+test = ["pytest>=8"]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["hannah_logging*"]

hannah_logging-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

hannah_logging-0.1.0/tests/test_buffer.py

@@ -0,0 +1,42 @@
+from hannah_logging.buffer import Entry, LogBuffer
+
+
+def entry(ts, message="x" * 36):
+    return Entry(timestamp_ms=ts, level=2, logger="", message=message, category=1)
+
+
+def test_take_returns_in_order():
+    buf = LogBuffer(max_bytes=10_000)
+    for ts in range(5):
+        buf.put(entry(ts))
+    gap, items = buf.take(max_items=3, timeout=0)
+    assert gap is None
+    assert [e.timestamp_ms for e in items] == [0, 1, 2]
+    _, items = buf.take(max_items=10, timeout=0)
+    assert [e.timestamp_ms for e in items] == [3, 4]
+
+
+def test_overflow_drops_oldest_and_records_gap():
+    size = entry(0).size()
+    buf = LogBuffer(max_bytes=size * 3)
+    for ts in range(10, 16):
+        buf.put(entry(ts))
+    gap, items = buf.take(max_items=10, timeout=0)
+    assert [e.timestamp_ms for e in items] == [13, 14, 15]
+    assert (gap.dropped, gap.from_ms, gap.to_ms) == (3, 10, 12)
+
+
+def test_gap_is_reported_once():
+    size = entry(0).size()
+    buf = LogBuffer(max_bytes=size)
+    buf.put(entry(1))
+    buf.put(entry(2))
+    gap, _ = buf.take(max_items=10, timeout=0)
+    assert gap.dropped == 1
+    gap, items = buf.take(max_items=10, timeout=0)
+    assert gap is None and items == []
+
+
+def test_take_times_out_when_empty():
+    buf = LogBuffer(max_bytes=100)
+    assert buf.take(max_items=1, timeout=0.01) == (None, [])

hannah_logging-0.1.0/tests/test_handler.py

@@ -0,0 +1,61 @@
+import logging
+
+import pytest
+from hannah_proto import logging_pb2
+
+from hannah_logging import CATEGORY_ATTR, GENERAL, METADATA, TRANSCRIPT
+from hannah_logging.buffer import LogBuffer
+from hannah_logging.handler import CollectorHandler
+from hannah_logging.secrets import SecretFilter
+
+
+@pytest.fixture
+def setup():
+    buf = LogBuffer(max_bytes=1_000_000)
+    handler = CollectorHandler(buf, SecretFilter(literals=["topsecret"]), {"app.stt": TRANSCRIPT})
+    logger = logging.getLogger("app")
+    logger.setLevel(logging.DEBUG)
+    logger.propagate = False
+    logger.addHandler(handler)
+    yield buf
+    logger.removeHandler(handler)
+
+
+def taken(buf):
+    return buf.take(max_items=100, timeout=0)[1]
+
+
+def test_record_fields(setup):
+    logging.getLogger("app.x").warning("hello %s", "world")
+    [e] = taken(setup)
+    assert e.message == "hello world"
+    assert e.logger == "app.x"
+    assert e.level == logging_pb2.LOG_LEVEL_WARNING
+    assert e.category == GENERAL
+
+
+def test_timestamp_is_record_creation_time(setup):
+    logger = logging.getLogger("app")
+    record = logger.makeRecord("app", logging.INFO, __file__, 1, "x", (), None)
+    record.created = 1_700_000_000.123
+    logger.handle(record)
+    [e] = taken(setup)
+    assert e.timestamp_ms == 1_700_000_000_123
+
+
+def test_categories(setup):
+    logging.getLogger("app.stt").info("heard something")
+    logging.getLogger("app.stt.sub").info("also transcript")
+    logging.getLogger("app.stt2").info("not a child")
+    logging.getLogger("app").info("room Küche", extra={CATEGORY_ATTR: METADATA})
+    assert [e.category for e in taken(setup)] == [TRANSCRIPT, TRANSCRIPT, GENERAL, METADATA]
+
+
+def test_secrets_masked_including_traceback(setup):
+    try:
+        raise ValueError("failed with topsecret")
+    except ValueError:
+        logging.getLogger("app").exception("login topsecret")
+    [e] = taken(setup)
+    assert "topsecret" not in e.message
+    assert "ValueError" in e.message

hannah_logging-0.1.0/tests/test_integration.py

@@ -0,0 +1,177 @@
+"""End to end against in-process fakes of Hannah Core (SubscribeInfrastructure) and the
+log collector (LogService.Ship)."""
+import logging
+import queue
+import time
+from concurrent import futures
+
+import grpc
+import pytest
+from hannah_proto import hannah_pb2_grpc, infrastructure_pb2, logging_pb2, logging_pb2_grpc
+
+import hannah_logging
+
+
+class FakeCollector(logging_pb2_grpc.LogServiceServicer):
+    def __init__(self):
+        self.hellos = []
+        self.entries = []
+        self.gaps = []
+        self.metadata = []
+
+    def Ship(self, request_iterator, context):
+        self.metadata.append(dict(context.invocation_metadata()))
+        accepted = 0
+        for msg in request_iterator:
+            which = msg.WhichOneof("payload")
+            if which == "hello":
+                self.hellos.append(msg.hello)
+            elif which == "entry":
+                self.entries.append(msg.entry)
+                accepted += 1
+            elif which == "gap":
+                self.gaps.append(msg.gap)
+        return logging_pb2.ShipAck(accepted=accepted)
+
+
+class FakeCore(hannah_pb2_grpc.HannahServiceServicer):
+    def __init__(self, collector_port):
+        self.collector_port = collector_port
+        self.metadata = []
+        self.events = queue.Queue()
+
+    def SubscribeInfrastructure(self, request, context):
+        self.metadata.append(dict(context.invocation_metadata()))
+        yield infrastructure_pb2.InfrastructureMessage(snapshot=infrastructure_pb2.InfrastructureSnapshot(services=[
+            infrastructure_pb2.ServiceEndpoint(
+                kind=infrastructure_pb2.SERVICE_KIND_LOG_COLLECTOR,
+                instance="main", host="127.0.0.1", port=self.collector_port,
+            ),
+        ]))
+        while context.is_active():
+            try:
+                yield self.events.get(timeout=0.1)
+            except queue.Empty:
+                pass
+
+
+def start_server(register):
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=8))
+    register(server)
+    port = server.add_insecure_port("127.0.0.1:0")
+    server.start()
+    return server, port
+
+
+def wait_for(predicate, timeout=10.0):
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if predicate():
+            return True
+        time.sleep(0.05)
+    return False
+
+
+@pytest.fixture
+def collector():
+    fake = FakeCollector()
+    server, port = start_server(lambda s: logging_pb2_grpc.add_LogServiceServicer_to_server(fake, s))
+    fake.port = port
+    yield fake
+    server.stop(None)
+
+
+@pytest.fixture
+def logger():
+    log = logging.getLogger("itest")
+    log.setLevel(logging.DEBUG)
+    log.propagate = False
+    yield log
+    for h in list(log.handlers):
+        log.removeHandler(h)
+
+
+def test_buffered_entries_arrive_via_discovery_with_original_timestamp(collector, logger):
+    core = FakeCore(collector.port)
+    core_server, core_port = start_server(lambda s: hannah_pb2_grpc.add_HannahServiceServicer_to_server(core, s))
+    try:
+        shipping = hannah_logging.install("testcomp", version="1.2.3", instance="pc1",
+                                          secrets=["s3cr3t-value"], logger=logger)
+        logger.info("before connect, password is s3cr3t-value")
+        logged_at_ms = int(time.time() * 1000)
+        time.sleep(0.3)  # the entry has to be older than the connect
+
+        shipping.connect(hannah_address=f"127.0.0.1:{core_port}")
+        assert wait_for(lambda: len(collector.entries) >= 1)
+
+        [entry] = collector.entries
+        assert abs(entry.timestamp_ms - logged_at_ms) < 100
+        assert "s3cr3t-value" not in entry.message
+        assert collector.hellos[0].component == "testcomp"
+        assert collector.hellos[0].instance == "pc1"
+        assert collector.hellos[0].version == "1.2.3"
+
+        logger.warning("after connect")
+        assert wait_for(lambda: len(collector.entries) == 2)
+        assert collector.entries[1].level == logging_pb2.LOG_LEVEL_WARNING
+
+        shipping.close()
+        assert core.metadata[0]["x-proto-version"]
+        assert collector.metadata[0]["x-proto-version"]
+    finally:
+        core_server.stop(None)
+
+
+def test_static_address_fallback_and_flush_on_close(collector, logger):
+    shipping = hannah_logging.install("testcomp", logger=logger,
+                                      collector_address=f"127.0.0.1:{collector.port}")
+    for i in range(50):
+        logger.info("line %d", i)
+    shipping.close(timeout=5)
+    assert [e.message for e in collector.entries] == [f"line {i}" for i in range(50)]
+
+
+def test_gap_reported_after_overflow_before_connect(collector, logger):
+    shipping = hannah_logging.install("testcomp", logger=logger, max_buffer_bytes=1000)
+    for i in range(100):
+        logger.info("overflowing line %03d", i)
+    shipping.connect(collector_address=f"127.0.0.1:{collector.port}")
+    assert wait_for(lambda: collector.gaps and collector.entries)
+    shipping.close(timeout=5)
+    dropped = collector.gaps[0].dropped
+    assert dropped > 0
+    assert dropped + len(collector.entries) == 100
+    assert collector.entries[-1].message == "overflowing line 099"
+
+
+def test_unreachable_collector_never_blocks_logging(logger):
+    shipping = hannah_logging.install("testcomp", logger=logger, collector_address="127.0.0.1:1")
+    start = time.monotonic()
+    for i in range(1000):
+        logger.info("line %d", i)
+    assert time.monotonic() - start < 1.0
+    shipping.close(timeout=0.5)
+
+
+def test_own_thread_records_are_not_buffered(logger):
+    lib_logger = logging.getLogger("hannah_logging")
+    seen = []
+
+    class Recorder(logging.Handler):
+        def emit(self, record):
+            seen.append(record)
+
+    recorder = Recorder()
+    lib_logger.addHandler(recorder)
+    shipping = hannah_logging.install("testcomp", logger=logger)
+    # Records from the library's own threads must not loop back into the buffer,
+    # even if the host routes them through the collector handler.
+    lib_logger.addHandler(shipping.handler)
+    try:
+        shipping.connect(hannah_address="127.0.0.1:1")  # unreachable -> discovery warns
+        assert wait_for(lambda: any(r.threadName == "hannah-logging-discovery" for r in seen))
+        assert len(shipping._buffer) == 0
+    finally:
+        lib_logger.removeHandler(shipping.handler)
+        lib_logger.removeHandler(recorder)
+        shipping.close(timeout=0.5)

hannah_logging-0.1.0/tests/test_secrets.py

@@ -0,0 +1,59 @@
+import pytest
+
+from hannah_logging.secrets import MASK, SecretFilter
+
+
+@pytest.fixture
+def f():
+    return SecretFilter()
+
+
+@pytest.mark.parametrize("message, secret", [
+    ("login password=hunter2 ok", "hunter2"),
+    ("bot_token: 123abc456def", "123abc456def"),
+    ('{"api_key": "abcd-efgh-ijkl"}', "abcd-efgh-ijkl"),
+    ("headers Authorization: Bearer eyJhbGciOi.payload.sig", "eyJhbGciOi.payload.sig"),
+    ("connect mqtt://hannah:s3cr3tpw@broker:1883", "s3cr3tpw"),
+    ("jwt eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.c2lnbmF0dXJl here", "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.c2lnbmF0dXJl"),
+    ("telegram 123456789:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsawE started", "AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsawE"),
+    ("using glpat-abcdefghij0123456789 for api", "glpat-abcdefghij0123456789"),
+    ("psk=0123456789abcdef", "0123456789abcdef"),
+])
+def test_builtin_patterns_mask_secret(f, message, secret):
+    out = f(message)
+    assert secret not in out
+    assert MASK in out
+
+
+@pytest.mark.parametrize("message", [
+    "llm answered, tokens=512",
+    "token_count=3 authorized=true",
+    "Licht Küche eingeschaltet",
+])
+def test_harmless_messages_untouched(f, message):
+    assert f(message) == message
+
+
+def test_private_key_block():
+    f = SecretFilter()
+    out = f("key:\n-----BEGIN RSA PRIVATE KEY-----\nMIIEow\nabc\n-----END RSA PRIVATE KEY-----\nrest")
+    assert "MIIEow" not in out
+    assert out.endswith("rest")
+
+
+def test_literal_secrets_and_add_literal():
+    f = SecretFilter(literals=["my-very-secret"])
+    assert f("value is my-very-secret!") == f"value is {MASK}!"
+    f.add_literal("added-later-123")
+    assert "added-later-123" not in f("x added-later-123 y")
+
+
+def test_short_literals_ignored():
+    # A 1–3 character "secret" would mangle ordinary text.
+    f = SecretFilter(literals=["ab"])
+    assert f("abc") == "abc"
+
+
+def test_custom_pattern():
+    f = SecretFilter(patterns=[r"SN-\d{6}"])
+    assert f("serial SN-123456") == f"serial {MASK}"