logreducer 3.4.0__py3-none-any.whl

logreducer/sources.py

@@ -0,0 +1,78 @@
+"""Input sources for logreducer.
+
+logreducer is a pure reduction engine over an abstraction: its input is any
+RE-ITERABLE iterable of ``str`` lines - a "source". ``__iter__`` must return a
+FRESH iterator each call, because some reduction modes make more than one pass
+over the input (hybrid reads it twice). A ``list[str]``, or a class whose
+``__iter__`` re-opens the underlying stream, satisfies this; a bare generator
+does not.
+
+The package does not own connections or the loading path. An application can
+hand the reducer its own iterable - a database cursor, a Kafka consumer, an
+open file - and the reducer never manages that IO. The adapters here are thin
+conveniences for the CLI and quick use: ``FileSource`` is built in; SQL /
+ClickHouse / Kafka sources live behind optional extras (see the sql, clickhouse
+and kafka submodules).
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterable, Iterator, Sequence
+from typing import Any, Protocol, runtime_checkable
+
+from .memory import MemoryMonitor, StreamingProcessor
+
+
+def rows_to_lines(rows: Iterable[Sequence[Any]]) -> Iterator[str]:
+    """Yield the first column of each row as a stripped, non-empty line.
+
+    The shared row -> line convention for the DB adapters (SQL, ClickHouse):
+    NULLs and blank lines are skipped, matching FileSource semantics, so the
+    same data reduces identically whichever source it arrives through.
+    """
+    for row in rows:
+        value = row[0]
+        if value is None:
+            continue
+        line = str(value).strip()
+        if line:
+            yield line
+
+
+@runtime_checkable
+class Source(Protocol):
+    """A re-iterable stream of log lines.
+
+    Any object whose ``__iter__`` yields ``str`` and can be iterated more than
+    once is a valid source - a plain ``list[str]`` included.
+    """
+
+    def __iter__(self) -> Iterator[str]: ...
+
+
+class FileSource:
+    """Stream stripped, non-empty lines from a text file.
+
+    Re-iterable and memory-aware: each iteration re-opens the file and streams
+    it through the constant-memory strategy (full / chunked / reservoir-sampled
+    by size), so the reducer can make multiple passes without loading the file.
+    """
+
+    def __init__(self, path: str | os.PathLike[str], *, max_memory_gb: float = 1.0) -> None:
+        self.path = os.fspath(path)
+        self._processor = StreamingProcessor(MemoryMonitor(max_memory_gb))
+
+    def __iter__(self) -> Iterator[str]:
+        return self._processor.read_file_streaming(self.path)
+
+    @property
+    def size_bytes(self) -> int | None:
+        """File size in bytes, or None if it cannot be determined."""
+        try:
+            return os.path.getsize(self.path)
+        except OSError:
+            return None
+
+    def __repr__(self) -> str:
+        return f"FileSource({self.path!r})"

logreducer/sql.py

@@ -0,0 +1,209 @@
+"""SQL input source for logreducer (optional ``sql`` extra).
+
+Streams log lines out of any SQL database SQLAlchemy can reach - PostgreSQL,
+MySQL, SQLite, and the rest - using a server-side cursor (the ``yield_per``
+execution option, which turns on ``stream_results``). Rows are pulled in
+batches, so memory stays flat regardless of how many rows the query returns;
+the reducer never holds the whole result set.
+
+This is a thin convenience over the abstraction, not a data-access layer. An
+application that already has a SQLAlchemy ``Engine`` passes it straight in and
+keeps ownership of it; the package only borrows a connection per pass. A URL
+string is also accepted for CLI use, in which case the engine is created and
+owned here (call ``close()`` or use the source as a context manager to dispose
+it).
+
+Row -> line convention: the query selects the log line as its **first column**.
+Anything else (levels, timestamps) should be folded into that column in SQL,
+e.g. ``SELECT ts || ' ' || msg AS line FROM events``.
+
+SQLAlchemy is the deliberate choice here: it is the only mature *cross-database*
+Python layer that streams via server-side cursors. Single-database drivers
+(asyncpg, psycopg) or columnar loaders (ConnectorX, Polars) either lock to one
+backend or materialise the entire result - the memory hazard this avoids.
+
+Install: ``pip install 'logreducer[sql]'`` plus your database's DBAPI driver
+(``psycopg`` for PostgreSQL, ``PyMySQL`` for MySQL; SQLite needs none).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from sqlalchemy.engine import Engine
+
+_INSTALL_HINT = (
+    "logreducer's SQL source needs SQLAlchemy. Install the extra:\n"
+    "    pip install 'logreducer[sql]'\n"
+    "plus your database's DBAPI driver (psycopg for PostgreSQL, PyMySQL for "
+    "MySQL; SQLite needs none)."
+)
+
+
+class SQLSource:
+    """A re-iterable stream of log lines from a SQL query.
+
+    Each iteration opens a fresh connection and re-runs the query with a
+    server-side cursor, so the reducer's multi-pass modes work (every pass is
+    an idempotent re-read). The first column of each row is the log line;
+    NULLs and blank lines are skipped, matching FileSource.
+    """
+
+    def __init__(
+        self,
+        connectable: Engine | str,
+        query: str,
+        *,
+        params: dict[str, Any] | None = None,
+        yield_per: int = 1000,
+        sample: float | None = None,
+        sample_seed: int | None = None,
+    ) -> None:
+        """Build a SQL source.
+
+        Args:
+            connectable: A SQLAlchemy ``Engine`` (borrowed, not disposed) or a
+                database URL string (an engine is created and owned here).
+            query: A SQL SELECT whose first column is the log line.
+            params: Optional bound parameters for the query.
+            yield_per: Server-side cursor batch size (rows fetched per round
+                trip). Larger trades memory for fewer round trips.
+            sample: Optional fraction in (0, 1] - return only that fraction of
+                rows via a per-dialect random predicate. Pass ``sample_seed``
+                for a deterministic, re-iterable sample (required for the
+                reducer's multi-pass modes; PostgreSQL/MySQL only). Without a
+                seed the sample is best-effort and not reproducible across passes.
+            sample_seed: Seed for a reproducible sample. Raises
+                ``SamplingNotSupported`` on engines with no seedable RNG (SQLite).
+        """
+        try:
+            from sqlalchemy import create_engine
+            from sqlalchemy.engine import Engine as _Engine
+        except ImportError as exc:  # pragma: no cover - exercised only without the extra
+            raise ImportError(_INSTALL_HINT) from exc
+
+        if isinstance(connectable, str):
+            self._engine = create_engine(connectable)
+            self._owns_engine = True
+        elif isinstance(connectable, _Engine):
+            self._engine = connectable
+            self._owns_engine = False
+        else:
+            raise TypeError(
+                f"connectable must be a SQLAlchemy Engine or a URL string, got {type(connectable).__name__}"
+            )
+
+        self.query = query
+        self.params = params
+        self.yield_per = yield_per
+        self.sample = sample
+        self.sample_seed = sample_seed
+
+        # Precompute the sampled SQL now so an unsupported request (e.g. a seed
+        # on SQLite) fails at construction, not deep inside iteration.
+        self._sample_setup: str | None = None
+        self._sample_sql: str | None = None
+        if sample is not None:
+            from .sampling import build_sample_sql
+
+            self._sample_setup, self._sample_sql = build_sample_sql(
+                self._engine.dialect.name, query, sample, sample_seed
+            )
+
+    @classmethod
+    def from_table(
+        cls,
+        connectable: Engine | str,
+        table: str,
+        column: str,
+        *,
+        sample: float,
+        where: str | None = None,
+        sample_seed: int | None = None,
+        method: str = "system",
+        yield_per: int = 1000,
+        params: dict[str, Any] | None = None,
+    ) -> SQLSource:
+        """Sample a fraction of a table with the engine's NATIVE sampler.
+
+        Unlike ``sample=`` on the constructor (a random predicate over an
+        arbitrary query, which full-scans), this builds ``SELECT column FROM
+        table TABLESAMPLE ...`` so PostgreSQL samples at the page level -
+        genuinely cheap on a large table - with ``REPEATABLE(seed)`` for
+        determinism. MySQL/SQLite have no TABLESAMPLE and fall back to a
+        full-scan predicate. ``method`` is ``system`` (page, fast) or
+        ``bernoulli`` (row, uniform) on PostgreSQL.
+        """
+        try:
+            from sqlalchemy import create_engine
+        except ImportError as exc:  # pragma: no cover - exercised only without the extra
+            raise ImportError(_INSTALL_HINT) from exc
+        from .sampling import build_table_sample_query
+
+        owns = isinstance(connectable, str)
+        engine = create_engine(connectable) if owns else connectable
+        preparer = engine.dialect.identifier_preparer
+        query = build_table_sample_query(
+            engine.dialect.name,
+            preparer.quote(table),
+            preparer.quote(column),
+            fraction=sample,
+            seed=sample_seed,
+            method=method,
+            where=where,
+        )
+        source = cls(engine, query, yield_per=yield_per, params=params)
+        source._owns_engine = owns
+        return source
+
+    def __iter__(self) -> Iterator[str]:
+        from sqlalchemy import text
+
+        from .sources import rows_to_lines
+
+        stmt = text(self._sample_sql if self._sample_sql is not None else self.query)
+        with self._engine.connect() as conn:
+            # A deterministic sample (PostgreSQL) needs its per-connection seed
+            # set before the query, on the same connection.
+            if self._sample_setup is not None:
+                conn.exec_driver_sql(self._sample_setup)
+            # yield_per implies stream_results: a server-side cursor fetching
+            # `yield_per` rows per round trip, so memory does not grow with the
+            # result size.
+            result = conn.execution_options(yield_per=self.yield_per).execute(stmt, self.params or {})
+            yield from rows_to_lines(result)
+
+    def sample_batch(self, n: int) -> list[str]:
+        """Return one fresh random batch of up to ``n`` log lines.
+
+        Each call runs ``ORDER BY <rand> LIMIT n`` over the query, so successive
+        calls draw different rows (with-replacement) - the primitive the
+        ``reduce_to_target`` loop pulls batches from. Not re-iterable; a one-off
+        list, not a streaming pass. Deliberately samples the FULL query
+        population, ignoring any constructor ``sample=`` fraction - the target
+        loop wants fresh draws from everything, not a sample of a sample.
+        """
+        from sqlalchemy import text
+
+        from .sampling import build_sample_batch_sql
+        from .sources import rows_to_lines
+
+        sql = build_sample_batch_sql(self._engine.dialect.name, self.query, n)
+        with self._engine.connect() as conn:
+            return list(rows_to_lines(conn.execute(text(sql), self.params or {})))
+
+    def close(self) -> None:
+        """Dispose the engine, but only if this source created it."""
+        if self._owns_engine:
+            self._engine.dispose()
+
+    def __enter__(self) -> SQLSource:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    def __repr__(self) -> str:
+        return f"SQLSource({self._engine.url!r}, query={self.query!r})"

logreducer/target.py

@@ -0,0 +1,164 @@
+"""Reduce-to-target orchestrator.
+
+Where ``LogReducer.reduce`` makes a single pass over a whole source, this pulls
+fresh random batches and reduces them one at a time, accumulating distinct
+representative lines until it has the number the caller asked for - or the
+source runs dry, a fetch cap is hit, or the representatives stop growing.
+
+Peak memory is bounded to roughly one batch plus the accumulator: batches are
+sized to a byte budget (from the observed average row size), and a psutil
+watchdog shrinks the batch and, as a hard backstop, stops the loop under
+pressure. This is the memory-safe path for "give me X lines from a huge table"
+without a full scan.
+
+Batches come from ``source.sample_batch(n)`` where the source supports it (SQL,
+ClickHouse); any other re-iterable falls back to a client-side reservoir sample
+over the source per round.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
+
+from .memory import MemoryMonitor
+from .sampling import estimate_batch_rows, reservoir_sample
+
+if TYPE_CHECKING:
+    from .core import LogReducer
+    from .sources import Source
+
+_DEFAULT_BATCH_ROWS = 10_000
+_MIN_BATCH_ROWS = 1_000
+
+
+@runtime_checkable
+class _Sampleable(Protocol):
+    """A source that can hand back a fresh random batch of ``n`` lines."""
+
+    def sample_batch(self, n: int) -> list[str]: ...
+
+
+def _pull_batch(source: Source, n: int, rng: random.Random) -> list[str]:
+    """One fresh random batch of up to ``n`` lines.
+
+    Uses the source's own ``sample_batch`` (SQL/ClickHouse do it in the engine);
+    otherwise reservoir-samples the whole re-iterable client-side - correct, but
+    it re-reads the source each round, so it is best for in-memory or file
+    sources rather than remote ones.
+    """
+    if isinstance(source, _Sampleable):
+        return source.sample_batch(n)
+    return reservoir_sample(iter(source), n, rng)
+
+
+def reduce_to_target(
+    source: Source,
+    reducer: LogReducer,
+    *,
+    target_rows: int,
+    max_fetches: int = 50,
+    max_batch_memory_gb: float | None = None,
+    batch_rows: int | None = None,
+    plateau_rounds: int = 2,
+    seed: int | None = None,
+) -> dict[str, Any]:
+    """Accumulate reduced representative lines until ``target_rows`` are found.
+
+    Args:
+        source: The data source. If it has ``sample_batch(n)`` (SQL, ClickHouse)
+            each round draws a fresh random batch from the engine; otherwise the
+            whole source is reservoir-sampled per round.
+        reducer: A configured ``LogReducer``. Reused across batches - its state
+            is reset per ``reduce()`` call, so each batch is reduced independently.
+        target_rows: How many distinct representative lines to collect.
+        max_fetches: Hard cap on batch pulls (backstop against a source that
+            never yields enough distinct representatives).
+        max_batch_memory_gb: Byte budget a batch must fit; batch size is derived
+            from the observed average row size. Also arms the memory watchdog.
+            Defaults to the reducer's own ``max_memory_gb``.
+        batch_rows: Fixed batch size; disables adaptive sizing when set.
+        plateau_rounds: Stop after this many consecutive batches that add no new
+            representatives (a low-cardinality source may never reach the target).
+        seed: Seed for the client-side reservoir fallback (ignored when the
+            source samples in the engine).
+
+    Returns:
+        ``{"lines": [...], "stats": {...}}`` - up to ``target_rows`` lines plus
+        counts and the stop reason (``target`` / ``exhausted`` / ``max_fetches``
+        / ``plateau`` / ``memory``).
+    """
+    if target_rows <= 0:
+        raise ValueError(f"target_rows must be positive, got {target_rows!r}")
+    if max_fetches <= 0:
+        raise ValueError(f"max_fetches must be positive, got {max_fetches!r}")
+    if batch_rows is not None and batch_rows <= 0:
+        raise ValueError(f"batch_rows must be positive, got {batch_rows!r}")
+    if plateau_rounds <= 0:
+        raise ValueError(f"plateau_rounds must be positive, got {plateau_rounds!r}")
+
+    # As documented: the byte budget defaults to the reducer's own memory cap,
+    # so adaptive batch sizing is on by default (pass batch_rows to fix it).
+    budget_gb = max_batch_memory_gb if max_batch_memory_gb is not None else reducer.config.max_memory_gb
+    monitor = MemoryMonitor(budget_gb)
+    budget_bytes = int(budget_gb * 1024**3)
+    rng = random.Random(seed)
+
+    seen: dict[str, None] = {}
+    batch = batch_rows if batch_rows is not None else _DEFAULT_BATCH_ROWS
+    fetches = 0
+    plateau = 0
+    reason = "max_fetches"
+
+    while len(seen) < target_rows and fetches < max_fetches:
+        lines = _pull_batch(source, batch, rng)
+        fetches += 1
+        if not lines:
+            reason = "exhausted"
+            break
+
+        # Size the next batch from the bytes we just observed (adaptive mode).
+        if budget_bytes and batch_rows is None:
+            batch = estimate_batch_rows(lines, budget_bytes)
+
+        added = 0
+        for line in reducer.reduce(lines):
+            if line not in seen:
+                seen[line] = None
+                added += 1
+                if len(seen) >= target_rows:
+                    break
+
+        if len(seen) >= target_rows:
+            reason = "target"
+            break
+
+        if added == 0:
+            plateau += 1
+            if plateau >= plateau_rounds:
+                reason = "plateau"
+                break
+        else:
+            plateau = 0
+
+        # Watchdog: soft-shrink the next batch under pressure; hard-stop if the
+        # limit is genuinely exceeded after a GC.
+        _, safe = monitor.check_memory()
+        if not safe:
+            batch = max(_MIN_BATCH_ROWS, batch // 2)
+            if monitor.is_limit_exceeded():
+                reason = "memory"
+                break
+
+    result = list(seen)[:target_rows]
+    return {
+        "lines": result,
+        "stats": {
+            "target_rows": target_rows,
+            "collected": len(result),
+            "fetches": fetches,
+            "final_batch_rows": batch,
+            "peak_memory_gb": monitor.get_peak_usage_gb(),
+            "stop_reason": reason,
+        },
+    }

logreducer/temporal.py

@@ -0,0 +1,163 @@
+"""
+Temporal processing and time-aware pattern extraction
+"""
+
+import re
+from collections import defaultdict
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+from drain3 import TemplateMiner
+
+
+@dataclass
+class LogEntry:
+    """Parsed log entry with timestamp"""
+
+    raw_line: str
+    timestamp: datetime | None = None
+    level: str | None = None
+    line_number: int = 0
+
+
+class TimestampParser:
+    """Parse timestamps from various formats"""
+
+    PATTERNS = [
+        (r"\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2}", "%Y-%m-%d %H:%M:%S"),
+        (r"\d{1,2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}", "%d/%b/%Y:%H:%M:%S"),
+        (r"\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}", "%b %d %H:%M:%S"),
+    ]
+
+    COMPILED_PATTERNS = [(re.compile(p), f) for p, f in PATTERNS]
+    LEVEL_PATTERN = re.compile(r"\b(DEBUG|INFO|WARN|ERROR|CRITICAL)\b", re.I)
+
+    def parse_line(self, line: str, line_number: int = 0) -> LogEntry:
+        """Parse log line"""
+        entry = LogEntry(raw_line=line.strip(), line_number=line_number)
+
+        # Extract timestamp
+        for pattern, date_format in self.COMPILED_PATTERNS:
+            match = pattern.search(line)
+            if match:
+                try:
+                    timestamp_str = match.group()
+                    if "%Y" not in date_format:
+                        timestamp_str = f"{datetime.now().year} {timestamp_str}"
+                        date_format = f"%Y {date_format}"
+                    entry.timestamp = datetime.strptime(timestamp_str.replace("T", " "), date_format)
+                    break
+                except (ValueError, TypeError):
+                    # Timestamp parsing failed, try next pattern
+                    pass
+
+        # Extract level
+        level_match = self.LEVEL_PATTERN.search(line)
+        if level_match:
+            entry.level = level_match.group().upper()
+
+        return entry
+
+
+class TemporalProcessor:
+    """Time-aware log processing"""
+
+    def __init__(self, window_minutes: int = 60):
+        self.window_minutes = window_minutes
+        self.parser = TimestampParser()
+        self.miners_by_window: dict[int, TemplateMiner] = {}
+
+    def process_temporal(self, lines: list[str]) -> dict:
+        """Process with temporal awareness"""
+        entries = [self.parser.parse_line(line, i) for i, line in enumerate(lines)]
+
+        timed = [e for e in entries if e.timestamp]
+        timeless = [e for e in entries if not e.timestamp]
+
+        results: dict[str, Any] = {
+            "temporal_patterns": [],
+            "timeless_patterns": [],
+            "time_distribution": {},
+        }
+
+        if timed:
+            results["temporal_patterns"] = self._extract_temporal_patterns(timed)
+            results["time_distribution"] = self._analyze_distribution(timed)
+
+        if timeless:
+            results["timeless_patterns"] = self._extract_timeless_patterns(timeless)
+
+        return results
+
+    def _extract_temporal_patterns(self, entries: list[LogEntry]) -> list[dict]:
+        """Extract patterns with time context"""
+        patterns = []
+        windows = defaultdict(list)
+
+        # Group by time window
+        for entry in entries:
+            if entry.timestamp:
+                window_key = int(entry.timestamp.timestamp() // (self.window_minutes * 60))
+                windows[window_key].append(entry)
+
+        # Process each window
+        for window_key, window_entries in windows.items():
+            if window_key not in self.miners_by_window:
+                from drain3.template_miner_config import TemplateMinerConfig
+
+                config = TemplateMinerConfig()
+                config.snapshot_interval_minutes = 0  # Disable snapshots
+                config.snapshot_compress_state = False
+                self.miners_by_window[window_key] = TemplateMiner(config=config)
+
+            miner = self.miners_by_window[window_key]
+
+            for entry in window_entries:
+                result = miner.add_log_message(entry.raw_line)
+                cluster_id = result["cluster_id"]
+
+                patterns.append(
+                    {
+                        "template": miner.drain.id_to_cluster[cluster_id].get_template(),
+                        "window": window_key,
+                        "count": 1,
+                        "example": entry.raw_line,
+                    }
+                )
+
+        return patterns
+
+    def _extract_timeless_patterns(self, entries: list[LogEntry]) -> list[dict]:
+        """Extract patterns without timestamps"""
+        from drain3.template_miner_config import TemplateMinerConfig
+
+        config = TemplateMinerConfig()
+        config.snapshot_interval_minutes = 0  # Disable snapshots
+        config.snapshot_compress_state = False
+        miner = TemplateMiner(config=config)
+        patterns = []
+
+        for entry in entries:
+            result = miner.add_log_message(entry.raw_line)
+            cluster_id = result["cluster_id"]
+            patterns.append(
+                {
+                    "template": miner.drain.id_to_cluster[cluster_id].get_template(),
+                    "example": entry.raw_line,
+                }
+            )
+
+        return patterns
+
+    def _analyze_distribution(self, entries: list[LogEntry]) -> dict:
+        """Analyze time distribution"""
+        dist: dict[str, Any] = {"hourly": defaultdict(int), "levels": defaultdict(int)}
+
+        for entry in entries:
+            if entry.timestamp:
+                dist["hourly"][entry.timestamp.hour] += 1
+            if entry.level:
+                dist["levels"][entry.level] += 1
+
+        return dist