logreducer 3.4.0__py3-none-any.whl

logreducer/__init__.py

@@ -0,0 +1,40 @@
+"""LogReducer - reduce large volumes of log lines to a representative sample.
+
+A streaming reduction engine (dedup -> pattern mining -> anomaly/temporal
+analysis) with an IO-agnostic core: any re-iterable stream of str lines is a
+valid Source. Ships as both a library and a `logreducer` CLI.
+
+Copyright 2026 HYPERI PTY LIMITED.
+Licensed under the Apache License, Version 2.0 (see LICENSE).
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("logreducer")
+except PackageNotFoundError:  # running from a source tree that is not installed
+    __version__ = "0.0.0+unknown"
+
+from .config import BigDialConfig, OutputFormat, ProcessingLevel, ProcessingMode
+from .core import LogReducer
+from .logging_config import setup_logging
+from .sampling import SamplingNotSupported
+from .sinks import FileSink, Sink
+from .sources import FileSource, Source
+from .target import reduce_to_target
+
+__all__ = [
+    "BigDialConfig",
+    "FileSink",
+    "FileSource",
+    "LogReducer",
+    "OutputFormat",
+    "ProcessingLevel",
+    "ProcessingMode",
+    "SamplingNotSupported",
+    "Sink",
+    "Source",
+    "__version__",
+    "reduce_to_target",
+    "setup_logging",
+]

logreducer/anomaly.py

@@ -0,0 +1,75 @@
+"""
+Anomaly detection utilities
+"""
+
+from loguru import logger
+
+# Try optional imports
+try:
+    from sklearn.ensemble import IsolationForest
+    from sklearn.feature_extraction.text import TfidfVectorizer
+
+    SKLEARN_AVAILABLE = True
+except ImportError:
+    SKLEARN_AVAILABLE = False
+
+
+class AnomalyDetector:
+    """Detect anomalous log lines"""
+
+    def __init__(self, contamination: float = 0.1):
+        self.contamination = contamination
+        self.enabled = SKLEARN_AVAILABLE
+
+        if not self.enabled:
+            logger.warning("scikit-learn not available, anomaly detection disabled")
+
+    def detect_anomalies(self, lines: list[str]) -> tuple[list[str], list[str]]:
+        """
+        Separate anomalous and normal lines
+
+        Returns:
+            (anomalous_lines, normal_lines)
+        """
+        if not self.enabled or len(lines) < 10:
+            return [], lines
+
+        # Vectorize
+        vectorizer = TfidfVectorizer(
+            max_features=min(1000, len(lines) // 10),
+            ngram_range=(1, 3),
+            max_df=0.9,
+            min_df=2 if len(lines) > 100 else 1,
+        )
+
+        try:
+            # Keep the TF-IDF matrix SPARSE - IsolationForest accepts sparse
+            # input directly. Densifying (.toarray()) would allocate
+            # n_lines x n_features x 8 bytes, which dwarfs the sparse matrix and
+            # can blow the memory budget on a large unique-line set.
+            X = vectorizer.fit_transform(lines)
+        except ValueError:
+            # Empty vocabulary (every term filtered by min_df/max_df) - nothing
+            # to score, so treat all lines as normal rather than failing.
+            return [], lines
+
+        iso_forest = IsolationForest(contamination=self.contamination, random_state=42, n_estimators=100)
+        labels = iso_forest.fit_predict(X)
+        scores = iso_forest.score_samples(X)
+
+        # One pass: split normal vs anomalous, collecting scores for the latter.
+        normal: list[str] = []
+        anomaly_with_scores: list[tuple[str, float]] = []
+        for line, label, score in zip(lines, labels, scores, strict=False):
+            if label == -1:
+                anomaly_with_scores.append((line, score))
+            else:
+                normal.append(line)
+
+        # Most anomalous first (IsolationForest scores: lower = more anomalous).
+        anomaly_with_scores.sort(key=lambda pair: pair[1])
+        anomalous = [line for line, _ in anomaly_with_scores]
+
+        logger.info(f"Found {len(anomalous)} anomalies out of {len(lines)} lines")
+
+        return anomalous, normal

logreducer/cli.py

@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+"""LogReducer command-line interface.
+
+A single command that reduces a log source to a representative sample. The
+source is one of:
+
+* a file - ``logreducer app.log``
+* a SQL or ClickHouse query - ``logreducer --dsn postgresql://... --query '...'``
+* a Kafka topic - ``logreducer --dsn kafka://broker:9092 --topic logs --group g``
+
+The DB and Kafka sources need the matching optional extra installed
+(``logreducer[sql]`` / ``[clickhouse]`` / ``[kafka]``); a plain file needs
+nothing. Built on typer.
+"""
+
+import json
+import sys
+from typing import Any
+
+import typer
+
+from . import __version__
+from .core import LogReducer
+from .logging_config import setup_logging
+from .sources import Source
+
+
+def _err(message: str) -> None:
+    """Write an error line to stderr."""
+    print(message, file=sys.stderr)
+
+
+def _version_callback(value: bool) -> None:
+    """Eager --version handler: print version and exit."""
+    if value:
+        print(f"logreducer version {__version__}")
+        raise typer.Exit(0)
+
+
+def _build_dsn_source(
+    dsn: str,
+    query: str | None,
+    topic: str | None,
+    group: str | None,
+    *,
+    sample: float | None = None,
+    sample_seed: int | None = None,
+) -> Source:
+    """Dispatch a ``--dsn`` to the matching source adapter by URL scheme.
+
+    ``clickhouse://`` -> ClickHouseSource, ``kafka://`` -> KafkaSource, and any
+    other scheme (postgresql, mysql, sqlite, ...) -> SQLSource via SQLAlchemy.
+    Adapter imports are lazy so a plain-file run never needs the extras.
+    ``sample`` (and ``sample_seed`` for SQL) apply to the DB sources; Kafka has
+    no sampling and rejects it.
+    """
+    scheme = dsn.split("://", 1)[0].lower() if "://" in dsn else ""
+
+    if scheme == "clickhouse":
+        if not query:
+            raise typer.BadParameter("--query is required for a clickhouse:// source")
+        from .clickhouse import ClickHouseSource
+
+        return ClickHouseSource(dsn, query, sample=sample)
+
+    if scheme == "kafka":
+        if not (topic and group):
+            raise typer.BadParameter("--topic and --group are required for a kafka:// source")
+        if sample is not None:
+            raise typer.BadParameter("--sample is not supported for a kafka:// source")
+        from .kafka import KafkaSource
+
+        brokers = dsn.split("://", 1)[1]
+        return KafkaSource(brokers, group, topic)
+
+    # Everything else is standard SQL reached through SQLAlchemy.
+    if not query:
+        raise typer.BadParameter("--query is required for a SQL --dsn source")
+    from .sql import SQLSource
+
+    return SQLSource(dsn, query, sample=sample, sample_seed=sample_seed)
+
+
+def _reduce(
+    input_file: str | None = typer.Argument(None, help="Log file to reduce (omit when using --dsn)"),
+    output: str | None = typer.Option(None, "--output", "-o", help="Output file (default: stdout)"),
+    output_format: str = typer.Option("line", "--format", help="Output format: line, json, jsonl"),
+    pretty_json: bool = typer.Option(False, "--pretty-json", help="Pretty-print JSON output"),
+    level: str = typer.Option("standard", "--level", "-l", help="Processing level: standard, enhanced, maximum"),
+    mode: str = typer.Option("pattern", "--mode", "-m", help="Processing mode: pattern, anomaly, temporal, hybrid"),
+    dsn: str | None = typer.Option(None, "--dsn", help="Source DSN: postgresql://, clickhouse://, kafka://, ..."),
+    query: str | None = typer.Option(None, "--query", help="SQL SELECT (first column is the log line) for --dsn"),
+    topic: str | None = typer.Option(None, "--topic", help="Kafka topic (with a kafka:// --dsn)"),
+    group: str | None = typer.Option(None, "--group", help="Kafka consumer group (with a kafka:// --dsn)"),
+    max_memory: float | None = typer.Option(None, "--max-memory", help="Maximum memory usage in GB"),
+    max_patterns: int | None = typer.Option(None, "--max-patterns", help="Maximum number of patterns to extract"),
+    log: bool = typer.Option(False, "--log", help="Enable processing logs"),
+    log_file: str | None = typer.Option(None, "--log-file", help="Log file path"),
+    log_level: str = typer.Option("INFO", "--log-level", help="Logging level: DEBUG, INFO, WARNING, ERROR"),
+    estimate: bool = typer.Option(False, "--estimate", help="Estimate processing requirements and exit (file only)"),
+    metadata: bool = typer.Option(False, "--metadata", help="Include detailed metadata in output"),
+    stats: bool = typer.Option(False, "--stats", help="Print processing statistics to stderr"),
+    sample: float | None = typer.Option(
+        None, "--sample", help="Sample this fraction (0-1) of source rows (SQL/ClickHouse)"
+    ),
+    sample_seed: int | None = typer.Option(
+        None, "--sample-seed", help="Seed for a reproducible --sample (SQL: PostgreSQL/MySQL)"
+    ),
+    target_rows: int | None = typer.Option(
+        None, "--target-rows", help="Collect this many reduced lines via repeated sampled batches"
+    ),
+    max_fetches: int = typer.Option(50, "--max-fetches", help="Max sampled batches to pull for --target-rows"),
+    max_batch_memory: float | None = typer.Option(
+        None, "--max-batch-memory", help="Per-batch memory budget in GB for --target-rows"
+    ),
+    version: bool = typer.Option(
+        False, "--version", "-V", callback=_version_callback, is_eager=True, help="Show version and exit"
+    ),
+) -> None:
+    """Reduce a log source to a representative sample."""
+    # Build reducer config overrides, dropping unset values.
+    overrides: dict[str, Any] = {
+        "max_memory_gb": max_memory,
+        "max_patterns": max_patterns,
+        "enable_logging": log or None,
+        "log_file": log_file,
+        "log_level": log_level,
+        "output_format": output_format,
+        "pretty_json": pretty_json or None,
+    }
+    kwargs = {k: v for k, v in overrides.items() if v is not None}
+
+    try:
+        reducer = LogReducer(level=level, mode=mode, **kwargs)
+    except ValueError as exc:
+        _err(f"Error: {exc}")
+        raise typer.Exit(1) from exc
+
+    # The reducer configures only a file sink; when --log is set the CLI also
+    # wants progress on the console.
+    if log:
+        setup_logging(enable=True, console=True, log_file=log_file, log_level=log_level)
+
+    # --estimate is a file-only, pre-flight sizing step.
+    if estimate:
+        if not input_file:
+            _err("Error: --estimate requires an input file")
+            raise typer.Exit(1)
+        _run_estimate(reducer, input_file)
+        return
+
+    # --target-rows switches to the reduce-to-target orchestrator: pull sampled
+    # batches and accumulate reduced lines until the target is met.
+    if target_rows is not None:
+        _run_target(
+            reducer,
+            dsn=dsn,
+            query=query,
+            topic=topic,
+            group=group,
+            input_file=input_file,
+            output=output,
+            output_format=output_format,
+            pretty_json=pretty_json,
+            stats=stats,
+            sample=sample,
+            sample_seed=sample_seed,
+            target_rows=target_rows,
+            max_fetches=max_fetches,
+            max_batch_memory=max_batch_memory,
+        )
+        return
+
+    try:
+        if dsn:
+            source = _build_dsn_source(dsn, query, topic, group, sample=sample, sample_seed=sample_seed)
+            try:
+                result = reducer.reduce(source, output_file=output, return_metadata=metadata)
+            finally:
+                close = getattr(source, "close", None)
+                if callable(close):
+                    close()
+        elif input_file:
+            if sample is not None or sample_seed is not None:
+                # Same treatment as the kafka:// rejection: fail loudly rather
+                # than silently ignore a flag that only applies to DB sources.
+                raise typer.BadParameter("--sample/--sample-seed apply to SQL/ClickHouse sources, not files")
+            result = reducer.process_file(input_file, output, return_metadata=metadata)
+        else:
+            _err("Error: provide a log file, or --dsn with --query (SQL/ClickHouse) or --topic/--group (Kafka)")
+            raise typer.Exit(1)
+    except (typer.Exit, typer.BadParameter):
+        raise  # our control-flow exit / typer usage errors - let typer format them
+    except KeyboardInterrupt:
+        _err("Processing interrupted by user")
+        raise typer.Exit(1) from None
+    except Exception as exc:
+        # File-not-found, a missing optional extra (ImportError), or an adapter
+        # failure (bad DSN, connection refused, bad SQL, Kafka error) - report a
+        # clean one-line error instead of dumping a Python traceback.
+        _err(f"Error: {exc}")
+        raise typer.Exit(1) from exc
+
+    _emit_result(result, output, output_format, pretty_json, metadata)
+
+    if stats:
+        _print_stats(reducer.stats)
+
+
+def _run_estimate(reducer: LogReducer, input_file: str) -> None:
+    """Print a pre-flight processing estimate for a file.
+
+    The estimate is the command's product, so it goes to stdout via ``print``
+    (not the library logger, which is disabled unless --log is passed).
+    """
+    try:
+        est = reducer.estimate_processing(input_file)
+    except OSError as exc:
+        _err(f"Error estimating processing: {exc}")
+        raise typer.Exit(1) from exc
+
+    print("Processing Estimation")
+    print("=" * 50)
+    print(f"File size: {est['file_size_gb']:.2f} GB")
+    print(f"Estimated memory: {est['memory_required_gb']:.2f} GB")
+    print(f"Processing strategy: {est['strategy']}")
+    print(f"Estimated time: {est['estimated_time_seconds']:.0f} seconds")
+    print(f"Will sample data: {'Yes' if est['will_sample'] else 'No'}")
+    print(f"Expected output lines: ~{est['estimated_output_lines']:,}")
+    if est["memory_required_gb"] > 8.0:
+        _err("Warning: large memory requirements detected; consider --max-memory")
+
+
+def _run_target(
+    reducer: LogReducer,
+    *,
+    dsn: str | None,
+    query: str | None,
+    topic: str | None,
+    group: str | None,
+    input_file: str | None,
+    output: str | None,
+    output_format: str,
+    pretty_json: bool,
+    stats: bool,
+    sample: float | None,
+    sample_seed: int | None,
+    target_rows: int,
+    max_fetches: int,
+    max_batch_memory: float | None,
+) -> None:
+    """Collect target_rows reduced lines by pulling repeated sampled batches."""
+    from .target import reduce_to_target
+
+    try:
+        if dsn:
+            source: Source = _build_dsn_source(dsn, query, topic, group, sample=sample, sample_seed=sample_seed)
+        elif input_file:
+            from .sources import FileSource
+
+            source = FileSource(input_file, max_memory_gb=reducer.config.max_memory_gb)
+        else:
+            _err("Error: --target-rows needs a log file, or --dsn with --query (SQL/ClickHouse)")
+            raise typer.Exit(1)
+
+        try:
+            outcome = reduce_to_target(
+                source,
+                reducer,
+                target_rows=target_rows,
+                max_fetches=max_fetches,
+                max_batch_memory_gb=max_batch_memory,
+                seed=sample_seed,
+            )
+        finally:
+            close = getattr(source, "close", None)
+            if callable(close):
+                close()
+    except (typer.Exit, typer.BadParameter):
+        raise
+    except KeyboardInterrupt:
+        _err("Processing interrupted by user")
+        raise typer.Exit(1) from None
+    except Exception as exc:
+        _err(f"Error: {exc}")
+        raise typer.Exit(1) from exc
+
+    _emit_lines(outcome["lines"], output, output_format, pretty_json)
+    if stats:
+        s = outcome["stats"]
+        print(
+            f"\nCollected {s['collected']}/{s['target_rows']} lines in {s['fetches']} fetch(es) "
+            f"(stopped: {s['stop_reason']})",
+            file=sys.stderr,
+        )
+
+
+def _emit_lines(lines: list[str], output: str | None, output_format: str, pretty_json: bool) -> None:
+    """Write reduced lines to a file (via FileSink) or print them to stdout."""
+    if output:
+        from .sinks import FileSink
+
+        FileSink(output, output_format=output_format).write(lines)
+        return
+    if output_format == "json":
+        print(json.dumps({"lines": lines}, indent=2 if pretty_json else None))
+    elif output_format == "jsonl":
+        for line in lines:
+            print(json.dumps({"line": line}))
+    else:
+        for line in lines:
+            print(line)
+
+
+def _emit_result(
+    result: list[str] | dict,
+    output: str | None,
+    output_format: str,
+    pretty_json: bool,
+    metadata: bool,
+) -> None:
+    """Print the reduced result to stdout when not writing to a file."""
+    if output:
+        return  # Already written by the reducer.
+
+    if metadata and isinstance(result, dict):
+        if output_format == "json":
+            print(json.dumps(result, indent=2 if pretty_json else None))
+        else:
+            for line in result["lines"]:
+                print(line)
+    elif isinstance(result, list):
+        # Honour --format on stdout the same way the --target-rows path does.
+        _emit_lines(result, None, output_format, pretty_json)
+
+
+def _print_stats(stats: dict) -> None:
+    """Print a processing summary to stderr (size/rate omitted for non-files)."""
+    print("\nProcessing completed:", file=sys.stderr)
+    input_size_mb = stats.get("input_size_mb")
+    if input_size_mb is not None:
+        print(f"  Input: {stats['input_lines']:,} lines ({input_size_mb:.1f} MB)", file=sys.stderr)
+    else:
+        print(f"  Input: {stats['input_lines']:,} lines", file=sys.stderr)
+    print(f"  Output: {stats['output_lines']:,} lines", file=sys.stderr)
+    print(f"  Reduction: {stats['reduction_percent']:.1f}%", file=sys.stderr)
+    print(f"  Time: {stats['processing_time_seconds']:.2f}s", file=sys.stderr)
+    rate = stats.get("processing_rate_mb_per_sec")
+    if rate is not None:
+        print(f"  Rate: {rate:.1f} MB/sec", file=sys.stderr)
+
+
+def main() -> None:
+    """Console-script entry point."""
+    typer.run(_reduce)
+
+
+if __name__ == "__main__":
+    main()

logreducer/clickhouse.py

@@ -0,0 +1,171 @@
+"""ClickHouse input source for logreducer (optional ``clickhouse`` extra).
+
+Streams log lines out of ClickHouse using the official ``clickhouse-connect``
+driver's native block streaming (``query_row_block_stream``). ClickHouse returns
+results in blocks; this iterates block by block and row by row, so memory stays
+flat no matter how large the result is.
+
+ClickHouse gets its own adapter rather than going through SQLAlchemy on purpose:
+ClickHouse's SQLAlchemy dialect buffers the whole result client-side, which
+defeats constant-memory streaming. The native driver's block interface is the
+only path that streams. Everything else mirrors SQLSource - re-iterable (each
+pass re-runs the query), first column is the log line, NULL/blank skipped.
+
+Row -> line convention: the query selects the log line as its **first column**,
+e.g. ``SELECT message FROM logs WHERE ...``.
+
+Install: ``pip install 'logreducer[clickhouse]'``.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from clickhouse_connect.driver.client import Client
+
+_INSTALL_HINT = "logreducer's ClickHouse source needs clickhouse-connect. Install the extra:\n    pip install 'logreducer[clickhouse]'"
+
+
+class ClickHouseSource:
+    """A re-iterable stream of log lines from a ClickHouse query.
+
+    Each iteration opens a block stream and re-runs the query, so the reducer's
+    multi-pass modes work. The first column of each row is the log line; NULLs
+    and blank lines are skipped, matching FileSource.
+    """
+
+    def __init__(
+        self,
+        client_or_dsn: Client | str,
+        query: str,
+        *,
+        parameters: dict[str, Any] | None = None,
+        settings: dict[str, Any] | None = None,
+        sample: float | None = None,
+    ) -> None:
+        """Build a ClickHouse source.
+
+        Args:
+            client_or_dsn: A ``clickhouse-connect`` Client (borrowed, not
+                closed) or a ``clickhouse://user:pass@host:port/db`` DSN string
+                (a client is created and owned here).
+            query: A SQL SELECT whose first column is the log line.
+            parameters: Optional query parameters (server-side binding).
+            settings: Optional ClickHouse settings for the query.
+            sample: Optional fraction in (0, 1] - appends ClickHouse's native
+                ``SAMPLE`` clause. ClickHouse sampling is deterministic (no seed
+                needed), so it is safe for the reducer's multi-pass modes. It
+                REQUIRES the queried table to declare ``SAMPLE BY`` in its
+                ORDER BY, and the query to be a plain ``SELECT ... FROM table``
+                (SAMPLE goes after the table, before any WHERE) - otherwise put
+                the SAMPLE clause in your own query instead.
+        """
+        try:
+            import clickhouse_connect
+            from clickhouse_connect.driver.client import Client as _Client
+        except ImportError as exc:  # pragma: no cover - exercised only without the extra
+            raise ImportError(_INSTALL_HINT) from exc
+
+        if isinstance(client_or_dsn, str):
+            self._client = clickhouse_connect.get_client(dsn=client_or_dsn)
+            self._owns_client = True
+        elif isinstance(client_or_dsn, _Client):
+            self._client = client_or_dsn
+            self._owns_client = False
+        else:
+            raise TypeError(
+                f"client_or_dsn must be a clickhouse-connect Client or a DSN string, got {type(client_or_dsn).__name__}"
+            )
+
+        if sample is not None:
+            if not (0.0 < sample <= 1.0):
+                raise ValueError(f"sample fraction must be in (0, 1], got {sample!r}")
+            # SAMPLE must sit after the table, BEFORE any WHERE/GROUP/ORDER -
+            # appending it to a query with those clauses builds invalid SQL, so
+            # fail fast and point at the alternatives.
+            if re.search(r"\b(WHERE|GROUP\s+BY|ORDER\s+BY|LIMIT)\b", query, re.IGNORECASE):
+                raise ValueError(
+                    "sample= only supports a plain 'SELECT ... FROM table' query "
+                    "(ClickHouse SAMPLE goes before WHERE/GROUP/ORDER); use "
+                    "from_table(..., where=...) or put SAMPLE in your own query"
+                )
+
+        self.query = query
+        self.parameters = parameters
+        self.settings = settings
+        self.sample = sample
+        self._query = f"{query} SAMPLE {float(sample)!r}" if sample is not None else query
+
+    @classmethod
+    def from_table(
+        cls,
+        client_or_dsn: Client | str,
+        table: str,
+        column: str,
+        *,
+        sample: float,
+        where: str | None = None,
+        parameters: dict[str, Any] | None = None,
+        settings: dict[str, Any] | None = None,
+    ) -> ClickHouseSource:
+        """Sample a fraction of a table with ClickHouse's native ``SAMPLE`` clause.
+
+        Builds ``SELECT column FROM table SAMPLE k [WHERE ...]`` with the clause
+        in the correct position (after the table, before WHERE). Deterministic by
+        construction. Requires the table to declare a ``SAMPLE BY`` key in its
+        ORDER BY; otherwise ClickHouse rejects the query.
+        """
+        if not (0.0 < sample <= 1.0):
+            raise ValueError(f"sample fraction must be in (0, 1], got {sample!r}")
+        # Backtick-quote identifiers, doubling any embedded backtick so a name
+        # can never break out of its quoting.
+        tbl = "`" + table.replace("`", "``") + "`"
+        col = "`" + column.replace("`", "``") + "`"
+        where_sql = f" WHERE {where}" if where else ""
+        query = f"SELECT {col} FROM {tbl} SAMPLE {float(sample)!r}{where_sql}"
+        return cls(client_or_dsn, query, parameters=parameters, settings=settings)
+
+    def __iter__(self) -> Iterator[str]:
+        from .sources import rows_to_lines
+
+        with self._client.query_row_block_stream(
+            self._query, parameters=self.parameters, settings=self.settings
+        ) as stream:
+            for block in stream:
+                yield from rows_to_lines(block)
+
+    def sample_batch(self, n: int) -> list[str]:
+        """Return one fresh random batch of up to ``n`` log lines.
+
+        Runs ``... ORDER BY rand() LIMIT n`` over the query, so successive calls
+        draw different rows (with-replacement) - the primitive the
+        ``reduce_to_target`` loop pulls from. A one-off list, not a stream.
+        Deliberately samples the FULL query population, ignoring any constructor
+        ``sample=`` fraction - the target loop wants fresh draws from everything.
+        """
+        from .sampling import build_sample_batch_sql
+        from .sources import rows_to_lines
+
+        sql = build_sample_batch_sql("clickhouse", self.query, n)
+        out: list[str] = []
+        with self._client.query_row_block_stream(sql, parameters=self.parameters, settings=self.settings) as stream:
+            for block in stream:
+                out.extend(rows_to_lines(block))
+        return out
+
+    def close(self) -> None:
+        """Close the client, but only if this source created it."""
+        if self._owns_client:
+            self._client.close()
+
+    def __enter__(self) -> ClickHouseSource:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    def __repr__(self) -> str:
+        return f"ClickHouseSource(query={self.query!r})"