logreducer 3.4.0__py3-none-any.whl

logreducer/memory.py

@@ -0,0 +1,247 @@
+"""
+Memory management and monitoring utilities
+"""
+
+import gc
+import os
+import random
+from collections import deque
+from collections.abc import Iterable, Iterator
+from typing import Any
+
+import psutil
+
+
+class MemoryMonitor:
+    """Monitor and control memory usage during processing"""
+
+    def __init__(self, max_memory_gb: float = 1.0):
+        self.max_memory_gb = max_memory_gb
+        self.max_memory_bytes = max_memory_gb * 1024 * 1024 * 1024
+        self.effective_limit = self.max_memory_bytes * 0.8  # 80% safety margin
+        self.process = psutil.Process()
+        self.peak_usage_bytes = 0
+
+        # Initialize peak usage
+        current_usage = self.process.memory_info().rss
+        self.peak_usage_bytes = current_usage
+
+        # Calculate safe parameters
+        avg_line_size = 200  # bytes
+        python_overhead = 3
+        self.max_lines_in_memory = int(self.effective_limit / (avg_line_size * python_overhead))
+        self.safe_chunk_size = max(1000, self.max_lines_in_memory // 10)
+
+    def check_memory(self) -> tuple[float, bool]:
+        """Return (current_gb, is_safe) against the SOFT 80% threshold.
+
+        Not a pure getter: when the soft threshold is crossed this runs a
+        gc.collect() and re-tests, so transient garbage does not trigger
+        degradation. Callers use this as the soft tier and
+        ``is_limit_exceeded`` (the full limit) as the hard tier.
+        """
+        current = self.process.memory_info().rss
+        current_gb = current / (1024**3)
+        is_safe = current < self.effective_limit
+
+        # Update peak usage
+        if current > self.peak_usage_bytes:
+            self.peak_usage_bytes = current
+
+        if not is_safe:
+            gc.collect()
+            current = self.process.memory_info().rss
+            current_gb = current / (1024**3)
+            is_safe = current < self.effective_limit
+
+        return current_gb, is_safe
+
+    def get_current_usage_gb(self) -> float:
+        """Get current memory usage in GB"""
+        current = self.process.memory_info().rss
+        # Update peak usage
+        if current > self.peak_usage_bytes:
+            self.peak_usage_bytes = current
+        return float(current / (1024**3))
+
+    def get_peak_usage_gb(self) -> float:
+        """Get peak memory usage in GB"""
+        # Update current peak if needed
+        current = self.process.memory_info().rss
+        if current > self.peak_usage_bytes:
+            self.peak_usage_bytes = current
+        return self.peak_usage_bytes / (1024**3)
+
+    def is_limit_exceeded(self) -> bool:
+        """True when RSS exceeds the FULL configured limit (the hard tier).
+
+        Deliberately compares against ``max_memory_bytes``, not the 80%
+        ``effective_limit`` used by ``check_memory`` - the gap between the two
+        is what gives callers room to soft-degrade (shrink batches, sample)
+        before the hard stop fires.
+        """
+        current = self.process.memory_info().rss
+        return bool(current > self.max_memory_bytes)
+
+    def reset(self) -> None:
+        """Reset peak memory tracking"""
+        current_usage = self.process.memory_info().rss
+        self.peak_usage_bytes = current_usage
+
+    def estimate_file_strategy(self, file_size_bytes: int) -> str:
+        """Determine processing strategy"""
+        file_size_gb = file_size_bytes / (1024**3)
+
+        if file_size_gb < self.max_memory_gb * 0.2:
+            return "full"
+        elif file_size_gb < self.max_memory_gb * 5:
+            return "chunked"
+        else:
+            return "sampled"
+
+
+class StreamingProcessor:
+    """Process large files with constant memory usage"""
+
+    def __init__(self, memory_monitor: MemoryMonitor):
+        self.memory_monitor = memory_monitor
+        self.chunk_size = memory_monitor.safe_chunk_size
+
+    def read_file_streaming(self, file_path: str) -> Iterator[str]:
+        """Read file with minimal memory usage"""
+        file_size = os.path.getsize(file_path)
+        strategy = self.memory_monitor.estimate_file_strategy(file_size)
+
+        if strategy == "full":
+            with open(file_path, encoding="utf-8", errors="ignore") as f:
+                for line in f:
+                    line = line.strip()
+                    if line:
+                        yield line
+
+        elif strategy == "chunked":
+            yield from self._read_chunked(file_path)
+
+        else:  # sampled
+            yield from self._read_sampled(file_path)
+
+    def _read_chunked(self, file_path: str) -> Iterator[str]:
+        """Read file in chunks, degrading to sampling under memory pressure.
+
+        The memory check runs on its own line counter, NOT on chunk-flush
+        boundaries - a flush count and a line count almost never coincide, so
+        gating the check on both used to make this fallback effectively dead.
+        """
+        chunk: list[str] = []
+        check_every = 100_000
+        next_check = check_every
+
+        with open(file_path, encoding="utf-8", errors="ignore") as f:
+            for line_num, raw in enumerate(line.strip() for line in f):
+                if not raw:
+                    continue
+
+                chunk.append(raw)
+
+                if len(chunk) >= self.chunk_size:
+                    yield from chunk
+                    chunk = []
+
+                if line_num >= next_check:
+                    next_check += check_every
+                    _, is_safe = self.memory_monitor.check_memory()
+                    if not is_safe:
+                        yield from chunk
+                        yield from self._read_sampled_remainder(f)
+                        return
+
+            if chunk:
+                yield from chunk
+
+    def _read_sampled(self, file_path: str) -> Iterator[str]:
+        """Reservoir sampling for huge files (uniform, reproducible).
+
+        Uses Algorithm L with a fixed-seed RNG so every pass over this
+        re-iterable FileSource yields the SAME sample - the reducer's multi-pass
+        modes need a stable input. (The old ``hash(line) % (i+1)`` reservoir was
+        neither uniform nor stable across processes.)
+        """
+        from .sampling import reservoir_sample
+
+        reservoir_size = min(self.chunk_size, 100000)
+
+        def _lines() -> Iterator[str]:
+            with open(file_path, encoding="utf-8", errors="ignore") as f:
+                for line in f:
+                    stripped = line.strip()
+                    if stripped:
+                        yield stripped
+
+        yield from reservoir_sample(_lines(), reservoir_size, random.Random(0))
+
+    def _read_sampled_remainder(self, file_handle: Any) -> Iterator[str]:
+        """Reservoir-sample the rest of an already-open file (mid-chunk fallback)."""
+        from .sampling import reservoir_sample
+
+        def _lines() -> Iterator[str]:
+            for line in file_handle:
+                stripped = line.strip()
+                if stripped:
+                    yield stripped
+
+        yield from reservoir_sample(_lines(), 10000, random.Random(0))
+
+
+class BoundedDeduplicator:
+    """Memory-bounded exact deduplication (sliding-window semantics).
+
+    The seen-set is capped at ``max_cache_size`` hashes with FIFO eviction, so
+    on very high-cardinality input a line can re-emit after its hash has been
+    evicted - bounded memory is traded for global dedup guarantees.
+    """
+
+    def __init__(self, max_cache_size: int = 100000, hash_algorithm: str = "xxhash"):
+        self.max_cache_size = max_cache_size
+
+        # Choose hash function
+        if hash_algorithm == "xxhash":
+            try:
+                import xxhash
+
+                self.hash_func = lambda x: xxhash.xxh64(x.encode()).hexdigest()
+            except ImportError:
+                import hashlib
+
+                self.hash_func = lambda x: hashlib.blake2b(x.encode(), digest_size=16).hexdigest()
+        else:
+            import hashlib
+
+            self.hash_func = lambda x: hashlib.md5(x.encode(), usedforsecurity=False).hexdigest()
+
+        self.seen_hashes: deque[str] = deque(maxlen=max_cache_size)
+        self.seen_set: set[str] = set()
+        self.stats = {"total": 0, "unique": 0, "duplicates": 0}
+
+    def is_duplicate(self, line: str) -> bool:
+        """Check if line is duplicate"""
+        self.stats["total"] += 1
+        line_hash = self.hash_func(line)
+
+        if line_hash in self.seen_set:
+            self.stats["duplicates"] += 1
+            return True
+
+        if len(self.seen_hashes) >= self.max_cache_size:
+            old_hash = self.seen_hashes[0]
+            self.seen_set.discard(old_hash)
+
+        self.seen_hashes.append(line_hash)
+        self.seen_set.add(line_hash)
+        self.stats["unique"] += 1
+        return False
+
+    def deduplicate_lines(self, lines: Iterable[str]) -> Iterator[str]:
+        """Deduplicate a stream of lines"""
+        for line in lines:
+            if not self.is_duplicate(line):
+                yield line

logreducer/patterns.py

@@ -0,0 +1,154 @@
+"""Pattern extraction (Drain3 template mining) and fuzzy deduplication."""
+
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from drain3 import TemplateMiner
+from drain3.template_miner_config import TemplateMinerConfig
+from loguru import logger
+
+if TYPE_CHECKING:
+    from .config import BigDialConfig
+
+# datasketch powers fuzzy dedup and ships in the optional `enhanced` extra;
+# without it FuzzyDeduplicator degrades to a pass-through (with a warning).
+try:
+    from datasketch import MinHash, MinHashLSH
+
+    MINHASH_AVAILABLE = True
+except ImportError:
+    MINHASH_AVAILABLE = False
+
+
+@dataclass
+class LogPattern:
+    """Represents a unique log pattern."""
+
+    template: str
+    examples: list[str] = field(default_factory=list)
+    count: int = 0
+    priority: float = 0.0
+    anomaly_score: float = 0.0
+    metadata: dict = field(default_factory=dict)
+
+
+class PatternExtractor:
+    """Extract patterns using Drain3's online template miner."""
+
+    def __init__(self, config: "BigDialConfig") -> None:
+        self.config = config
+        self.setup_drain3()
+
+    def setup_drain3(self) -> None:
+        """Configure Drain3."""
+        drain_config = TemplateMinerConfig()
+        drain_config.profiling_enabled = False
+        drain_config.drain_sim_th = self.config.drain_similarity
+        drain_config.drain_depth = 4
+        drain_config.snapshot_interval_minutes = 0  # Disable snapshots
+        drain_config.snapshot_compress_state = False
+        # Bound the template store when configured: Drain3 LRU-evicts beyond
+        # drain_max_clusters, keeping memory flat on high-cardinality logs.
+        drain_config.drain_max_clusters = self.config.max_clusters
+        self.miner = TemplateMiner(config=drain_config)
+
+    def extract_patterns(self, lines: Iterable[str]) -> list[LogPattern]:
+        """Extract patterns from lines (accepts a stream; only clusters are held).
+
+        Feeds each line into Drain3's online miner, so the caller can pass a
+        generator - no need to materialise the whole line set. Memory is bounded
+        by the number of clusters (see ``max_clusters``), not the line count.
+        """
+        pattern_map = {}
+
+        for line in lines:
+            result = self.miner.add_log_message(line)
+            cluster_id = result["cluster_id"]
+
+            if cluster_id not in pattern_map:
+                cluster = self.miner.drain.id_to_cluster[cluster_id]
+                pattern = LogPattern(template=cluster.get_template(), count=0)
+                pattern_map[cluster_id] = pattern
+
+            pattern = pattern_map[cluster_id]
+            pattern.count += 1
+
+            if len(pattern.examples) < self.config.examples_per_pattern:
+                pattern.examples.append(line)
+
+        patterns = list(pattern_map.values())
+
+        # Apply filters
+        patterns = self._filter_by_occurrence(patterns)
+        patterns = self._calculate_priority(patterns)
+
+        # Sort by priority and limit
+        patterns.sort(key=lambda p: p.priority, reverse=True)
+        return patterns[: self.config.max_patterns]
+
+    def _filter_by_occurrence(self, patterns: list[LogPattern]) -> list[LogPattern]:
+        """Filter patterns by minimum occurrence."""
+        return [p for p in patterns if p.count >= self.config.min_pattern_occurrences]
+
+    def _calculate_priority(self, patterns: list[LogPattern]) -> list[LogPattern]:
+        """Calculate priority scores.
+
+        Severity boosts are checked highest-first so a template containing both
+        (e.g. "CRITICAL: write FAILED") scores as critical, not merely error.
+        """
+        for pattern in patterns:
+            priority: float = pattern.count
+
+            template_upper = pattern.template.upper()
+            if "CRITICAL" in template_upper or "FATAL" in template_upper:
+                priority *= 200
+            elif "ERROR" in template_upper or "FAIL" in template_upper:
+                priority *= 100
+            elif "WARN" in template_upper:
+                priority *= 50
+
+            # Boost complex patterns
+            priority *= 1 + pattern.template.count("<*>") * 0.5
+            priority *= 1 + len(pattern.template.split()) * 0.1
+
+            pattern.priority = priority
+
+        return patterns
+
+
+class FuzzyDeduplicator:
+    """Fuzzy (near-duplicate) deduplication using MinHash LSH."""
+
+    def __init__(self, threshold: float = 0.8):
+        self.threshold = threshold
+        self.enabled = MINHASH_AVAILABLE
+
+        if self.enabled:
+            self.lsh = MinHashLSH(threshold=threshold, num_perm=64)
+        else:
+            logger.warning("datasketch not installed - fuzzy dedup disabled (pip install 'logreducer[enhanced]')")
+
+    def deduplicate_stream(self, lines: Iterable[str]) -> Iterator[str]:
+        """Yield near-unique lines as they arrive (streaming near-dup filter).
+
+        Queries the LSH per line and inserts only new ones, so the reducer never
+        materialises the full unique-line list. Note the LSH itself grows with
+        the number of near-unique lines - that is inherent to fuzzy dedup.
+        """
+        if not self.enabled:
+            yield from lines
+            return
+
+        for i, line in enumerate(lines):
+            m = MinHash(num_perm=64)
+            for word in line.split():
+                m.update(word.encode("utf8"))
+
+            if not self.lsh.query(m):
+                self.lsh.insert(f"line_{i}", m)
+                yield line
+
+    def deduplicate(self, lines: Iterable[str]) -> list[str]:
+        """Remove near-duplicates, returning a list (eager wrapper of the stream)."""
+        return list(self.deduplicate_stream(lines))

logreducer/py.typed

@@ -0,0 +1 @@
+# Marker file for PEP 561 - this package supports type checking

logreducer/sampling.py

@@ -0,0 +1,211 @@
+"""Sampling helpers: dialect-aware SQL sampling, reservoir sampling, batch sizing.
+
+Three concerns, all pure and server-free so they unit-test without a database:
+
+* ``build_sample_sql`` - wrap an arbitrary user query in a deterministic,
+  per-dialect random predicate so a fraction of rows is returned reproducibly
+  (needed because ``TABLESAMPLE`` only applies to a base table, not the
+  arbitrary ``SELECT`` a Source is given). Mirrors ibis: raise when a seed
+  cannot be honoured deterministically rather than fake reproducibility.
+* ``build_sample_batch_sql`` - one fresh random batch of ``n`` rows
+  (``ORDER BY <rand> LIMIT n``) for the reduce-to-target loop (with-replacement).
+* ``reservoir_sample`` (Algorithm L) and ``estimate_batch_rows`` - the
+  client-side primitives the target orchestrator and the memory watchdog use.
+"""
+
+from __future__ import annotations
+
+import math
+import random
+from collections.abc import Iterable
+from itertools import islice
+
+
+class SamplingNotSupported(Exception):
+    """A sampling request an engine cannot honour (e.g. a seed on SQLite)."""
+
+
+# Per-dialect random function, used by ORDER BY <rand> for fresh batches.
+RANDOM_FN: dict[str, str] = {
+    "postgresql": "random()",
+    "mysql": "rand()",
+    "mariadb": "rand()",
+    "sqlite": "random()",
+    "clickhouse": "rand()",
+}
+
+
+def _pg_sample(query: str, fraction: float, seed: int | None) -> tuple[str | None, str]:
+    # setseed makes random() deterministic for the rest of the connection; it
+    # takes a value in [-1, 1]. Each pass opens a fresh connection and re-seeds,
+    # so the sample is identical across passes (re-iterable). Caveat: strict
+    # identity also depends on PostgreSQL visiting rows in the same order -
+    # synchronized_seqscans on a busy table can rotate the scan start; ORDER BY
+    # in the query if bit-exact re-reads matter.
+    setup = None
+    if seed is not None:
+        s = ((seed % 2_000_000) / 1_000_000.0) - 1.0
+        setup = f"SELECT setseed({s!r})"
+    return setup, f"SELECT * FROM ({query}) AS _lr_sample WHERE random() < {float(fraction)!r}"
+
+
+def _mysql_sample(query: str, fraction: float, seed: int | None) -> tuple[str | None, str]:
+    # RAND(seed) with a constant seed is a repeatable per-row sequence.
+    rand = f"rand({int(seed)})" if seed is not None else "rand()"
+    return None, f"SELECT * FROM ({query}) AS _lr_sample WHERE {rand} < {float(fraction)!r}"
+
+
+def _sqlite_sample(query: str, fraction: float, seed: int | None) -> tuple[str | None, str]:
+    if seed is not None:
+        raise SamplingNotSupported(
+            "sqlite has no seedable RNG, so seeded (reproducible) sampling is unsupported; "
+            "omit sample_seed for best-effort sampling, or use a seedable engine (postgresql, mysql)"
+        )
+    threshold = int(float(fraction) * 1_000_000)
+    return None, f"SELECT * FROM ({query}) AS _lr_sample WHERE (abs(random()) % 1000000) < {threshold}"
+
+
+# dialect.name -> builder. ClickHouse is handled by its own source (native
+# SAMPLE clause), so it is intentionally not here.
+DIALECT_SAMPLERS = {
+    "postgresql": _pg_sample,
+    "mysql": _mysql_sample,
+    "mariadb": _mysql_sample,
+    "sqlite": _sqlite_sample,
+}
+
+
+def _check_fraction(fraction: float) -> None:
+    if not (0.0 < fraction <= 1.0):
+        raise ValueError(f"sample fraction must be in (0, 1], got {fraction!r}")
+
+
+def build_sample_sql(dialect: str, query: str, fraction: float, seed: int | None = None) -> tuple[str | None, str]:
+    """Wrap ``query`` so it returns ~``fraction`` of its rows, deterministically.
+
+    Returns ``(setup_sql, sampled_sql)`` where ``setup_sql`` is an optional
+    per-connection statement to run first (PostgreSQL ``setseed``), or None.
+
+    Raises ``ValueError`` for a bad fraction and ``SamplingNotSupported`` for a
+    dialect with no sampler or a seed the dialect cannot honour.
+    """
+    _check_fraction(fraction)
+    builder = DIALECT_SAMPLERS.get(dialect)
+    if builder is None:
+        raise SamplingNotSupported(f"no SQL sampler for dialect {dialect!r} (supported: {sorted(DIALECT_SAMPLERS)})")
+    return builder(query, fraction, seed)
+
+
+def build_sample_batch_sql(dialect: str, query: str, n: int) -> str:
+    """A fresh random batch of up to ``n`` rows (``ORDER BY <rand> LIMIT n``).
+
+    Deliberately unseeded: each call returns a different sample (with-replacement),
+    which is what the reduce-to-target loop wants.
+    """
+    if n <= 0:
+        raise ValueError(f"batch size must be positive, got {n!r}")
+    rand = RANDOM_FN.get(dialect)
+    if rand is None:
+        raise SamplingNotSupported(f"no random function for dialect {dialect!r} (supported: {sorted(RANDOM_FN)})")
+    return f"SELECT * FROM ({query}) AS _lr_batch ORDER BY {rand} LIMIT {int(n)}"
+
+
+def build_table_sample_query(
+    dialect: str,
+    table: str,
+    column: str,
+    *,
+    fraction: float,
+    seed: int | None = None,
+    method: str = "system",
+    where: str | None = None,
+) -> str:
+    """Build a native table-sampling query (``table``/``column`` already quoted).
+
+    PostgreSQL uses page-level ``TABLESAMPLE SYSTEM`` (or row-level ``BERNOULLI``)
+    with ``REPEATABLE(seed)`` - genuinely sub-linear, unlike the full-scan random
+    predicate used for arbitrary queries. Engines without TABLESAMPLE (MySQL,
+    SQLite) fall back to a predicate on the table (still a full scan).
+
+    Raises ``SamplingNotSupported`` for a dialect with no table sampler or a seed
+    it cannot honour, and ``ValueError`` for a bad fraction or method.
+    """
+    _check_fraction(fraction)
+    conds = [where] if where else []
+
+    if dialect == "postgresql":
+        m = method.lower()
+        if m not in ("system", "bernoulli"):
+            raise ValueError(f"method must be 'system' or 'bernoulli', got {method!r}")
+        pct = float(fraction) * 100.0
+        repeatable = f" REPEATABLE ({int(seed)})" if seed is not None else ""
+        where_sql = f" WHERE {' AND '.join(conds)}" if conds else ""
+        return f"SELECT {column} FROM {table} TABLESAMPLE {m.upper()} ({pct!r}){repeatable}{where_sql}"
+
+    # No TABLESAMPLE on these - a predicate on the table (full scan, but native).
+    if dialect in ("mysql", "mariadb"):
+        pred = f"rand({int(seed)}) < {float(fraction)!r}" if seed is not None else f"rand() < {float(fraction)!r}"
+    elif dialect == "sqlite":
+        if seed is not None:
+            raise SamplingNotSupported("sqlite has no seedable RNG; seeded table sampling is unsupported")
+        pred = f"(abs(random()) % 1000000) < {int(float(fraction) * 1_000_000)}"
+    else:
+        raise SamplingNotSupported(f"no table sampler for dialect {dialect!r}")
+
+    conds.append(pred)
+    return f"SELECT {column} FROM {table} WHERE {' AND '.join(conds)}"
+
+
+def reservoir_sample(items: Iterable[str], k: int, rng: random.Random) -> list[str]:
+    """Uniform sample of ``k`` items from a stream of unknown length (Algorithm L).
+
+    Without replacement, O(k) memory, O(k(1 + log(n/k))) expected time. Replaces
+    the old ``hash(line) % (i+1)`` reservoir, which was neither uniform nor
+    reproducible (it depended on PYTHONHASHSEED). Seed ``rng`` for reproducibility.
+    """
+    if k <= 0:
+        return []
+    it = iter(items)
+    reservoir = list(islice(it, k))
+    if len(reservoir) < k:
+        return reservoir  # fewer than k items - keep them all
+
+    def _u() -> float:
+        # Clamp away from 0.0 and 1.0 so log() and log1p(-w) never blow up.
+        return min(max(rng.random(), 1e-12), 1.0 - 1e-12)
+
+    w = math.exp(math.log(_u()) / k)
+    while True:
+        skip = math.floor(math.log(_u()) / math.log1p(-w))
+        # islice(it, skip, skip + 1) discards `skip` items and yields the next.
+        nxt = next(islice(it, skip, skip + 1), None)
+        if nxt is None:
+            return reservoir
+        reservoir[rng.randrange(k)] = nxt
+        w *= math.exp(math.log(_u()) / k)
+
+
+def estimate_batch_rows(
+    sample_lines: list[str],
+    budget_bytes: int,
+    *,
+    floor: int = 1000,
+    ceil: int = 1_000_000,
+    align: int = 1000,
+    overhead: int = 3,
+) -> int:
+    """Rows that fit ``budget_bytes``, from the average byte size of a sample.
+
+    ``overhead`` approximates Python's per-object memory multiplier (matching
+    MemoryMonitor). Result is clamped to ``[floor, ceil]`` and rounded down to a
+    multiple of ``align`` so it lines up with the cursor fetch batch.
+    """
+    if not sample_lines or budget_bytes <= 0:
+        return floor
+    avg_bytes = sum(len(s.encode("utf-8")) for s in sample_lines) / len(sample_lines)
+    per_row = max(1.0, avg_bytes * overhead)
+    rows = int(budget_bytes / per_row)
+    if align > 1:
+        rows = (rows // align) * align
+    # Clamp LAST so alignment can never push the result outside [floor, ceil].
+    return max(floor, min(ceil, rows))

logreducer/sinks.py

@@ -0,0 +1,81 @@
+"""Output sinks for logreducer.
+
+The reduction engine returns its result in memory (a ``list[str]``) - that is
+the primary, dependency-free output. A *sink* is the abstraction for sending
+those reduced lines somewhere else: a file, a Kafka topic, a database. Like
+``Source`` on the input side, a sink is a thin seam - the package does not own
+the connection or the delivery guarantees; an application can pass its own.
+
+``Sink`` is a structural protocol: anything with a ``write(lines) -> int`` is a
+sink. ``FileSink`` is built in; the Kafka producer sink lives behind the
+``kafka`` optional extra (see the kafka submodule).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Iterable
+from datetime import datetime
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class Sink(Protocol):
+    """A destination for reduced log lines.
+
+    ``write`` consumes an iterable of ``str`` lines and returns the number of
+    lines written. It may be called with a generator, so implementations should
+    iterate lazily rather than materialising the whole batch.
+    """
+
+    def write(self, lines: Iterable[str]) -> int: ...
+
+
+class FileSink:
+    """Write reduced lines to a text file in one of three formats.
+
+    A standalone, format-aware writer (``line`` / ``json`` / ``jsonl``) that an
+    application can use without touching the reducer. It streams the ``line``
+    and ``jsonl`` formats a row at a time (constant memory); ``json`` buffers
+    the list because the format is a single document.
+
+    This is deliberately simpler than the reducer's own ``output_file`` path,
+    which additionally writes a ``.meta.json`` sidecar of run stats - that needs
+    the reducer's context, whereas a sink only sees the lines.
+    """
+
+    def __init__(self, path: str | os.PathLike[str], *, output_format: str = "line") -> None:
+        self.path = os.fspath(path)
+        fmt = output_format.lower()
+        if fmt not in ("line", "json", "jsonl"):
+            raise ValueError(f"Unknown output_format {output_format!r} (use line, json or jsonl)")
+        self.output_format = fmt
+
+    def write(self, lines: Iterable[str]) -> int:
+        output_path = Path(self.path)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        count = 0
+        if self.output_format == "json":
+            # A JSON document is a single value - the list has to be built.
+            materialised = list(lines)
+            count = len(materialised)
+            with open(output_path, "w", encoding="utf-8") as f:
+                json.dump({"lines": materialised, "timestamp": datetime.now().isoformat()}, f, indent=2)
+        elif self.output_format == "jsonl":
+            with open(output_path, "w", encoding="utf-8") as f:
+                for line in lines:
+                    json.dump({"line": line}, f)
+                    f.write("\n")
+                    count += 1
+        else:  # line
+            with open(output_path, "w", encoding="utf-8") as f:
+                for line in lines:
+                    f.write(line + "\n")
+                    count += 1
+        return count
+
+    def __repr__(self) -> str:
+        return f"FileSink({self.path!r}, output_format={self.output_format!r})"