logreducer 3.4.0__py3-none-any.whl

logreducer/kafka.py

@@ -0,0 +1,300 @@
+"""Kafka input source and output sink for logreducer (optional ``kafka`` extra).
+
+Consume log lines from a Kafka topic to reduce them, and/or produce the reduced
+lines back to a topic. Built on ``confluent-kafka`` (librdkafka).
+
+The client defaults (``PRODUCER_DEFAULTS`` / ``CONSUMER_DEFAULTS``, librdkafka
+config names), the ``merge_config`` / credential-masking idioms and the
+``KafkaConsumerError`` / ``_PARTITION_EOF`` handling follow standard
+production Kafka client practice, so a config authored for another
+librdkafka-based client behaves the same here.
+
+Two intentional consumer design choices, because a logreducer ``Source`` must
+be finite and re-iterable (the reducer counts lines by exhausting the source,
+and re-reads it for multi-pass modes):
+
+* **Bounded.** The read stops when every assigned partition has reached its end
+  (``enable.partition.eof`` -> ``_PARTITION_EOF``), which is the deterministic
+  "caught up with the topic" signal. A consecutive-idle-poll count is only a
+  backstop for the never-assigned case (e.g. a missing topic).
+* **No commit.** Offsets are never committed, so each pass re-reads from the
+  earliest offset - that is what makes the source re-iterable. An application
+  that needs offset tracking should consume itself and hand the reducer its own
+  iterable of ``str``.
+
+Install: ``pip install 'logreducer[kafka]'``.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import time
+from collections.abc import Iterable, Iterator
+from typing import Any
+
+# Production-lean librdkafka defaults: durable produces, bounded timeouts,
+# cheap batching. Any key can be overridden by the user config.
+PRODUCER_DEFAULTS: dict[str, Any] = {
+    "acks": "all",  # Wait for all replicas (durability)
+    "retries": 5,  # Retry on transient failures
+    "retry.backoff.ms": 100,  # Backoff between retries
+    "delivery.timeout.ms": 120000,  # 2 minutes max delivery time
+    "request.timeout.ms": 30000,  # 30 seconds per request
+    "linger.ms": 5,  # Small delay for batching
+    "compression.type": "lz4",  # Fast compression
+    "batch.size": 16384,  # 16KB batch size
+}
+
+CONSUMER_DEFAULTS: dict[str, Any] = {
+    "auto.offset.reset": "earliest",  # Start from beginning if no offset
+    "enable.auto.commit": False,  # Manual control (this source never commits)
+    "session.timeout.ms": 45000,  # 45 seconds session timeout
+    "heartbeat.interval.ms": 3000,  # 3 seconds heartbeat
+    "max.poll.interval.ms": 300000,  # 5 minutes max poll interval
+    "fetch.min.bytes": 1,  # Return immediately with any data
+    "fetch.wait.max.ms": 500,  # Max wait for fetch.min.bytes
+    # logreducer-specific: needed so a drained partition reports EOF, which is
+    # how the bounded read knows it has caught up with the topic.
+    "enable.partition.eof": True,
+}
+
+_CREDENTIAL_KEYS: frozenset[str] = frozenset(
+    {
+        "sasl.password",
+        "sasl.username",
+        "ssl.key.password",
+        "ssl.keystore.password",
+        "ssl.truststore.password",
+    }
+)
+
+_INSTALL_HINT = (
+    "logreducer's Kafka source/sink needs confluent-kafka. Install the extra:\n    pip install 'logreducer[kafka]'"
+)
+
+
+def merge_config(user_config: dict[str, Any], defaults: dict[str, Any], verify_ssl: bool = True) -> dict[str, Any]:
+    """Overlay ``user_config`` on ``defaults``; optionally disable TLS verify.
+
+    User values win over defaults.
+    """
+    merged = {**defaults, **user_config}
+    if not verify_ssl:
+        merged["enable.ssl.certificate.verification"] = "false"
+    return merged
+
+
+def mask_credentials(config: dict[str, Any]) -> dict[str, Any]:
+    """Return a copy of ``config`` with credential values replaced by ``***``.
+
+    Used in ``__repr__`` so a Kafka config never leaks secrets into logs.
+    """
+    return {k: ("***" if k in _CREDENTIAL_KEYS and v not in (None, "") else v) for k, v in config.items()}
+
+
+class KafkaConsumerError(Exception):
+    """A fatal Kafka consumer error, carrying the broker error code if known."""
+
+    def __init__(self, message: str, error_code: int | None = None) -> None:
+        self.error_code = error_code
+        super().__init__(message)
+
+
+def _normalise_config(config: str | dict[str, Any]) -> dict[str, Any]:
+    """A bare string is treated as ``bootstrap.servers``; a dict is copied."""
+    if isinstance(config, str):
+        return {"bootstrap.servers": config}
+    return dict(config)
+
+
+class KafkaSource:
+    """A bounded, re-iterable stream of log lines from a Kafka topic.
+
+    Each iteration creates a fresh consumer, subscribes, and reads until every
+    assigned partition reports end-of-partition (i.e. the topic is drained),
+    decoding each message value to a ``str`` line. Offsets are never committed,
+    so every pass re-reads from the earliest offset.
+    """
+
+    def __init__(
+        self,
+        config: str | dict[str, Any],
+        group_id: str,
+        topics: str | list[str],
+        *,
+        verify_ssl: bool = True,
+        max_messages: int | None = None,
+        poll_timeout: float = 1.0,
+        idle_polls: int = 3,
+        assignment_timeout: float = 30.0,
+        encoding: str = "utf-8",
+    ) -> None:
+        """Build a Kafka source.
+
+        Args:
+            config: ``bootstrap.servers`` string, or a full librdkafka config.
+            group_id: Consumer group id.
+            topics: A topic name or list of names to subscribe to.
+            verify_ssl: If False, disable TLS certificate verification.
+            max_messages: Optional hard cap on lines read per pass.
+            poll_timeout: Seconds to block per ``poll`` call.
+            idle_polls: Backstop for a never-EOF broker - once partitions are
+                assigned, stop after this many consecutive empty polls.
+            assignment_timeout: Backstop for the never-assigned case (unreachable
+                broker, missing topic, auth failure). If no partition is assigned
+                within this many seconds, iteration raises instead of hanging.
+            encoding: Text encoding for decoding message values.
+        """
+        if importlib.util.find_spec("confluent_kafka") is None:  # pragma: no cover - only without the extra
+            raise ImportError(_INSTALL_HINT)
+
+        config = _normalise_config(config)
+        config["group.id"] = group_id
+        self._config = merge_config(config, CONSUMER_DEFAULTS, verify_ssl=verify_ssl)
+        self.topics = [topics] if isinstance(topics, str) else list(topics)
+        self.max_messages = max_messages
+        self.poll_timeout = poll_timeout
+        self.idle_polls = idle_polls
+        self.assignment_timeout = assignment_timeout
+        self.encoding = encoding
+
+    def __iter__(self) -> Iterator[str]:
+        from confluent_kafka import Consumer, KafkaError
+
+        consumer = Consumer(self._config)
+        consumer.subscribe(self.topics)
+        eof_partitions: set[tuple[str, int]] = set()
+        empty_polls = 0
+        emitted = 0
+        # Never-assigned backstop: an unreachable broker, a missing topic, or an
+        # auth failure leaves assignment() empty forever, so empty_polls (gated
+        # on assignment) never fires and the loop would spin indefinitely. Bail
+        # if no partition is ever assigned within the timeout - a stuck
+        # dependency surfaced as a real error, not a silent hang.
+        ever_assigned = False
+        assignment_deadline = time.monotonic() + self.assignment_timeout
+        try:
+            while True:
+                if self.max_messages is not None and emitted >= self.max_messages:
+                    break
+
+                msg = consumer.poll(self.poll_timeout)
+
+                if consumer.assignment():
+                    ever_assigned = True
+                elif not ever_assigned and time.monotonic() >= assignment_deadline:
+                    raise KafkaConsumerError(
+                        f"no partition assignment within {self.assignment_timeout:.0f}s for "
+                        f"topics {self.topics!r} (broker unreachable, missing topic, or auth failure?)"
+                    )
+
+                if msg is None:
+                    # Before partitions are assigned an empty poll just means the
+                    # group is still joining - don't count it. Once assigned,
+                    # repeated empties are the backstop for a never-EOF broker.
+                    if consumer.assignment():
+                        empty_polls += 1
+                        if empty_polls >= self.idle_polls:
+                            break
+                    continue
+                empty_polls = 0
+
+                error = msg.error()
+                if error:
+                    if error.code() == KafkaError._PARTITION_EOF:
+                        topic, partition = msg.topic(), msg.partition()
+                        if topic is not None and partition is not None:
+                            eof_partitions.add((topic, partition))
+                        assigned = {(tp.topic, tp.partition) for tp in consumer.assignment()}
+                        # Drained: every partition we hold has reported its end.
+                        if assigned and eof_partitions >= assigned:
+                            break
+                        continue
+                    raise KafkaConsumerError(f"Kafka error: {error.str()}", error_code=error.code())
+
+                value = msg.value()
+                if value is None:
+                    continue
+                line = value.decode(self.encoding, errors="replace").strip()
+                if line:
+                    emitted += 1
+                    yield line
+        finally:
+            consumer.close()
+
+    def __repr__(self) -> str:
+        return f"KafkaSource(topics={self.topics!r}, config={mask_credentials(self._config)!r})"
+
+
+class KafkaSink:
+    """A Sink that produces reduced lines to a Kafka topic.
+
+    ``write`` produces each line as a UTF-8 message value and flushes at the
+    end, returning the number of lines produced. Uses the standard
+    produce/poll/flush shape over ``PRODUCER_DEFAULTS``.
+    """
+
+    def __init__(
+        self,
+        config: str | dict[str, Any],
+        topic: str,
+        *,
+        key: str | None = None,
+        verify_ssl: bool = True,
+        flush_timeout: float | None = None,
+        encoding: str = "utf-8",
+    ) -> None:
+        """Build a Kafka sink.
+
+        Args:
+            config: ``bootstrap.servers`` string, or a full librdkafka config.
+            topic: Target topic for produced lines.
+            key: Optional fixed message key applied to every line.
+            verify_ssl: If False, disable TLS certificate verification.
+            flush_timeout: Seconds to wait on the final flush (None = infinite).
+            encoding: Text encoding for the produced message values.
+        """
+        try:
+            from confluent_kafka import Producer
+        except ImportError as exc:  # pragma: no cover - exercised only without the extra
+            raise ImportError(_INSTALL_HINT) from exc
+
+        self._config = merge_config(_normalise_config(config), PRODUCER_DEFAULTS, verify_ssl=verify_ssl)
+        self.topic = topic
+        self.key = key
+        self.flush_timeout = flush_timeout
+        self.encoding = encoding
+        self._producer = Producer(self._config)
+
+    def write(self, lines: Iterable[str]) -> int:
+        """Produce each line to the topic and flush; return the number produced.
+
+        With the default ``flush_timeout=None`` the final flush blocks until the
+        queue drains, so the count is effectively a delivery count. With a finite
+        ``flush_timeout`` some messages may still be in flight when flush returns.
+        """
+        key_bytes = self.key.encode(self.encoding) if self.key is not None else None
+        count = 0
+        for line in lines:
+            self._producer.produce(self.topic, value=line.encode(self.encoding), key=key_bytes)
+            self._producer.poll(0)  # Trigger delivery callbacks (non-blocking)
+            count += 1
+        self.flush()
+        return count
+
+    def flush(self) -> int:
+        """Wait for outstanding messages; return the count still in queue."""
+        if self.flush_timeout is not None:
+            result: int = self._producer.flush(self.flush_timeout)
+        else:
+            result = self._producer.flush()
+        return result
+
+    def __enter__(self) -> KafkaSink:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.flush()
+
+    def __repr__(self) -> str:
+        return f"KafkaSink(topic={self.topic!r}, config={mask_credentials(self._config)!r})"

logreducer/logging_config.py

@@ -0,0 +1,193 @@
+"""Logging configuration for LogReducer.
+
+Output follows one strict standard so logreducer's logs sit uniformly beside a
+host application's logs in a shared deployment: RFC 3339 timestamps, a
+``time | LEVEL | name:function:line - message`` layout, ASCII-only in files,
+solarized colours in an interactive terminal, a plain prefix in CI, and one
+JSON object per line when ``LOG_FORMAT=json``.
+
+logreducer is a library, not a service, so it only ever manages its OWN loguru
+handlers (filtered to ``logreducer`` records) and never removes a host
+application's handlers - embedding logreducer inside another application must
+not disturb that application's logging.
+
+ENV overrides:
+    LOG_LEVEL=DEBUG
+    LOG_FORMAT=json|text
+    LOG_OUTPUT=stdout|stderr
+"""
+
+import contextlib
+import os
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from loguru import logger
+
+if TYPE_CHECKING:
+    from loguru import Logger
+
+# Solarized palette (https://ethanschoonover.com/solarized/) for the console
+# sink - readable on both light and dark terminal backgrounds.
+SOLARIZED = {
+    "base01": "#586e75",
+    "green": "#859900",
+    "cyan": "#2aa198",
+    "blue": "#268bd2",
+    "yellow": "#b58900",
+    "orange": "#cb4b16",
+    "red": "#dc322f",
+}
+
+# Handler ids this module has registered, so re-configuring only tears down our
+# own sinks - never the host application's.
+_HANDLER_IDS: list[int] = []
+
+
+def _is_ci() -> bool:
+    """True in a CI environment (colours off, plain prefix format)."""
+    return (
+        os.getenv("CI") == "true"
+        or os.getenv("GITHUB_ACTIONS") == "true"
+        or os.getenv("GITLAB_CI") == "true"
+        or os.getenv("JENKINS_URL") is not None
+    )
+
+
+def _is_interactive() -> bool:
+    """True only for an interactive UTF-8 terminal (never a pipe/container)."""
+    if _is_ci() or not sys.stderr.isatty():
+        return False
+    term = os.getenv("TERM", "")
+    if not term or term == "dumb":
+        return False
+    locale = (os.getenv("LC_ALL") or os.getenv("LANG") or "").upper()
+    return "UTF-8" in locale or "UTF8" in locale
+
+
+def _console_format(colorize: bool, ci: bool) -> str:
+    """Console format string - solarized colours, or a plain CI prefix."""
+    if ci or not colorize:
+        return "[{level: <8}] {name}:{function}:{line} - {message}"
+    return (
+        f"<fg {SOLARIZED['green']}>{{time:YYYY-MM-DDTHH:mm:ss.SSSZZ}}</fg {SOLARIZED['green']}> | "
+        f"<level>{{level: <8}}</level> | "
+        f"<fg {SOLARIZED['cyan']}>{{name}}</fg {SOLARIZED['cyan']}>:"
+        f"<fg {SOLARIZED['cyan']}>{{function}}</fg {SOLARIZED['cyan']}>:"
+        f"<fg {SOLARIZED['cyan']}>{{line}}</fg {SOLARIZED['cyan']}> - "
+        f"<level>{{message}}</level>"
+    )
+
+
+def _file_format() -> str:
+    """File format string - ASCII only, RFC 3339 timestamp with bracketed level."""
+    return "{time:YYYY-MM-DDTHH:mm:ss.SSSZZ} [{level: <8}] {name}:{function}:{line} - {message}"
+
+
+def _apply_solarized_levels() -> None:
+    """Colour the loguru levels with the solarized scheme."""
+    logger.level("TRACE", color=f"<fg {SOLARIZED['base01']}>")
+    logger.level("DEBUG", color=f"<fg {SOLARIZED['base01']}>")
+    logger.level("INFO", color=f"<fg {SOLARIZED['blue']}>")
+    logger.level("SUCCESS", color=f"<fg {SOLARIZED['green']}>")
+    logger.level("WARNING", color=f"<fg {SOLARIZED['yellow']}>")
+    logger.level("ERROR", color=f"<fg {SOLARIZED['orange']}>")
+    logger.level("CRITICAL", color=f"<fg {SOLARIZED['red']}>")
+
+
+def setup_logging(
+    enable: bool = False,
+    log_file: str | None = None,
+    log_level: str = "INFO",
+    log_format: str = "text",
+    console: bool = False,
+    own_sinks: bool = True,
+) -> None:
+    """Configure logreducer's own logging sinks.
+
+    Args:
+        enable: Whether logreducer emits logs at all (default False).
+        log_file: Optional path for an ASCII, rotating file sink.
+        log_level: Threshold level (overridden by ``LOG_LEVEL``).
+        log_format: ``text`` (human) or ``json`` (overridden by ``LOG_FORMAT``).
+        console: Also log to the console (stderr, or ``LOG_OUTPUT=stdout``).
+        own_sinks: When False, register NO handlers - just enable the package
+            and let logreducer's records flow through the HOST application's
+            loguru handlers. This is the embedding seam: a host app that owns
+            logging calls ``setup_logging(enable=True, own_sinks=False)`` and
+            logreducer's output lands in the host's sinks, formatted by the
+            host's standard.
+    """
+    # Env beats caller, so a deployment can retune logging without code changes.
+    log_level = os.environ.get("LOG_LEVEL", log_level).upper()
+    fmt_selector = (os.environ.get("LOG_FORMAT") or log_format or "text").strip().lower()
+    serialize = fmt_selector == "json"
+
+    # Tear down ONLY our previously registered handlers.
+    global _HANDLER_IDS
+    for handler_id in _HANDLER_IDS:
+        try:
+            logger.remove(handler_id)
+        except ValueError:
+            pass
+    _HANDLER_IDS = []
+
+    if not enable:
+        logger.disable("logreducer")
+        return
+
+    if not own_sinks:
+        # Host-owned logging: enable the package, register nothing.
+        logger.enable("logreducer")
+        return
+
+    ci = _is_ci()
+    stream = sys.stdout if os.environ.get("LOG_OUTPUT") == "stdout" else sys.stderr
+
+    if console:
+        # CLI/app mode owns the console: drop loguru's default stderr handler
+        # (id 0) so our formatted line is not printed twice. Only the pristine
+        # default is removed - a host app's own handlers (ids > 0) are untouched
+        # (embedding logreducer only ever goes through the console=False path).
+        with contextlib.suppress(ValueError):
+            logger.remove(0)
+        colorize = not serialize and not ci and _is_interactive()
+        if colorize:
+            _apply_solarized_levels()
+        _HANDLER_IDS.append(
+            logger.add(
+                stream,
+                level=log_level,
+                format=_console_format(colorize, ci),
+                colorize=colorize,
+                serialize=serialize,
+                filter="logreducer",
+            )
+        )
+
+    if log_file:
+        try:
+            Path(log_file).parent.mkdir(parents=True, exist_ok=True)
+            _HANDLER_IDS.append(
+                logger.add(
+                    log_file,
+                    level=log_level,
+                    format=_file_format(),
+                    rotation="10 MB",
+                    retention="7 days",
+                    encoding="utf-8",
+                    serialize=serialize,
+                    filter="logreducer",
+                )
+            )
+        except OSError as exc:
+            # A bad log path should never take the process down.
+            logger.warning(f"Could not create log file {log_file}: {exc}")
+
+    logger.enable("logreducer")
+
+
+def get_logger(name: str = "logreducer") -> "Logger":
+    """Return a logger bound for the given logreducer submodule name."""
+    return logger.bind(name=name)