datablade 0.0.0__py3-none-any.whl → 0.0.6__py3-none-any.whl

datablade/dataframes/readers.py

@@ -0,0 +1,1367 @@
+"""
+Memory-aware file reading utilities with Polars support.
+
+This module provides intelligent file reading that:
+- Estimates memory requirements before loading
+- Automatically chunks large files
+- Uses Polars for high-performance reading when available
+- Writes large files to multiple Parquet partitions
+"""
+
+import csv
+import json
+import pathlib
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Callable, Iterable, Iterator, List, Optional, Union
+
+import pandas as pd
+import pyarrow.parquet as pq
+
+from ..utils.logging import (
+    build_log_context,
+    format_log_context,
+    log_debug,
+    log_info,
+    log_warning,
+    timed_step,
+)
+from ..utils.strings import coerce_path, ensure_directory
+
+if TYPE_CHECKING:
+    import polars as pl
+
+
+# File-size guardrails for text-based memory estimation.
+_TEXT_MEDIUM_FILE_BYTES = 100 * 1024 * 1024  # 100 MB
+_TEXT_LARGE_FILE_BYTES = 1024 * 1024 * 1024  # 1 GB
+_TEXT_EXTREME_FILE_BYTES = 5 * 1024 * 1024 * 1024  # 5 GB
+_TEXT_LARGE_MULTIPLIER = 4.0
+_TEXT_EXTREME_MULTIPLIER = 6.0
+
+
+# Heuristic thresholds for text file sizing (bytes) and multipliers used
+# when estimating memory requirements without sampling.
+# These are intentionally exported (leading underscore) so tests can
+# reference the same thresholds.
+_TEXT_MEDIUM_FILE_BYTES = 5 * 1024 * 1024
+_TEXT_LARGE_FILE_BYTES = 50 * 1024 * 1024
+_TEXT_EXTREME_FILE_BYTES = 1 * 1024 * 1024 * 1024
+
+_TEXT_LARGE_MULTIPLIER = 3.0
+_TEXT_EXTREME_MULTIPLIER = 10.0
+
+
+def _normalize_text_delimiter_kwargs(suffix: str, read_kwargs: dict) -> dict:
+    """Normalize delimiter/sep kwargs for delimited text formats.
+
+    - For TSV, default to tab separator unless the caller supplied one.
+    - For CSV/TXT, leave pandas defaults unless the caller supplied one.
+    """
+    if suffix not in (".csv", ".tsv", ".txt"):
+        return read_kwargs
+
+    if "sep" in read_kwargs or "delimiter" in read_kwargs:
+        return read_kwargs
+
+    if suffix == ".tsv":
+        out = dict(read_kwargs)
+        out["sep"] = "\t"
+        return out
+
+    return read_kwargs
+
+
+def _detect_text_encoding(
+    file_path: pathlib.Path,
+    sample_size: int = 10000,
+) -> Optional[str]:
+    """Detect file encoding using optional dependencies."""
+    try:
+        sample = file_path.read_bytes()[:sample_size]
+    except Exception:
+        return None
+
+    try:
+        from charset_normalizer import from_bytes
+
+        best = from_bytes(sample).best()
+        if best and best.encoding:
+            return best.encoding
+    except Exception:
+        pass
+
+    try:
+        import chardet
+
+        detected = chardet.detect(sample)
+        return detected.get("encoding")
+    except Exception:
+        return None
+
+
+def _detect_text_delimiter(sample_text: str) -> Optional[str]:
+    """Detect delimiter using csv.Sniffer with common delimiters."""
+    try:
+        sniffer = csv.Sniffer()
+        dialect = sniffer.sniff(sample_text, delimiters=[",", "\t", ";", "|"])
+        return dialect.delimiter
+    except Exception:
+        return None
+
+
+def _prepare_text_read_kwargs(
+    file_path: pathlib.Path,
+    suffix: str,
+    read_kwargs: dict,
+    verbose: bool,
+) -> dict:
+    """Apply optional encoding/delimiter detection for text formats."""
+    if suffix not in (".csv", ".tsv", ".txt"):
+        return dict(read_kwargs)
+
+    kwargs = dict(read_kwargs)
+    detect_encoding = bool(kwargs.pop("detect_encoding", False))
+    detect_delimiter = bool(kwargs.pop("detect_delimiter", False))
+
+    if detect_encoding and "encoding" not in kwargs:
+        detected_encoding = _detect_text_encoding(file_path)
+        if detected_encoding:
+            kwargs["encoding"] = detected_encoding
+            log_debug(
+                f"Detected encoding '{detected_encoding}' for {file_path.name}.",
+                verbose,
+            )
+
+    if detect_delimiter and "sep" not in kwargs and "delimiter" not in kwargs:
+        encoding = kwargs.get("encoding") or "utf-8"
+        try:
+            with open(file_path, "r", encoding=encoding, errors="replace") as handle:
+                sample_text = handle.read(8192)
+        except Exception:
+            sample_text = ""
+
+        detected_delimiter = _detect_text_delimiter(sample_text)
+        if detected_delimiter:
+            kwargs["sep"] = detected_delimiter
+            log_debug(
+                f"Detected delimiter '{detected_delimiter}' for {file_path.name}.",
+                verbose,
+            )
+
+    return kwargs
+
+
+def _polars_scan_csv_kwargs(suffix: str, read_kwargs: dict) -> dict:
+    """Best-effort mapping of pandas-style kwargs to polars scan_csv kwargs."""
+    # Polars uses `separator` (not `sep`). We only map delimiters because other
+    # pandas kwargs are not generally compatible.
+    scan_kwargs: dict = {}
+
+    if "sep" in read_kwargs:
+        scan_kwargs["separator"] = read_kwargs["sep"]
+    elif "delimiter" in read_kwargs:
+        scan_kwargs["separator"] = read_kwargs["delimiter"]
+    elif suffix == ".tsv":
+        scan_kwargs["separator"] = "\t"
+
+    if "has_header" in read_kwargs:
+        scan_kwargs["has_header"] = read_kwargs["has_header"]
+    elif "header" in read_kwargs:
+        header = read_kwargs["header"]
+        if header is None:
+            scan_kwargs["has_header"] = False
+        elif header == "infer" or header == 0:
+            scan_kwargs["has_header"] = True
+        elif isinstance(header, int):
+            scan_kwargs["has_header"] = True
+            if header > 0:
+                scan_kwargs["skip_rows"] = header
+
+    if "infer_schema_length" in read_kwargs:
+        scan_kwargs["infer_schema_length"] = read_kwargs["infer_schema_length"]
+
+    if "encoding" in read_kwargs:
+        scan_kwargs["encoding"] = read_kwargs["encoding"]
+
+    dtype_value = read_kwargs.get("dtypes", read_kwargs.get("dtype"))
+    if isinstance(dtype_value, (dict, list, tuple)):
+        scan_kwargs["dtypes"] = dtype_value
+
+    return scan_kwargs
+
+
+def _polars_scan_source(path: pathlib.Path, suffix: str, read_kwargs: dict):
+    """Create a Polars LazyFrame for formats with scan support."""
+    import polars as pl
+
+    if suffix == ".parquet":
+        return pl.scan_parquet(path)
+    if suffix in (".csv", ".tsv", ".txt"):
+        return pl.scan_csv(path, **_polars_scan_csv_kwargs(suffix, read_kwargs))
+    if suffix in (".json", ".jsonl") and read_kwargs.get("lines"):
+        return pl.scan_ndjson(path)
+    return None
+
+
+def _normalized_chunks(
+    chunks: Iterable[pd.DataFrame],
+    *,
+    convert_types: bool,
+    verbose: bool,
+) -> Iterator[pd.DataFrame]:
+    """Normalize chunk schemas and optionally coerce numeric strings."""
+    from .frames import clean_dataframe_columns, try_cast_string_columns_to_numeric
+
+    for chunk in chunks:
+        chunk = clean_dataframe_columns(chunk, verbose=verbose)
+        if convert_types:
+            chunk = try_cast_string_columns_to_numeric(chunk, verbose=verbose)
+        yield chunk
+
+
+def _iter_json_objects(
+    file_path: pathlib.Path,
+    record_path: str,
+) -> Iterator[dict]:
+    """Yield JSON objects from a standard JSON file using ijson."""
+    try:
+        import ijson
+    except ImportError as exc:
+        raise ImportError(
+            "Streaming non-JSON Lines files requires the optional 'ijson' dependency."
+        ) from exc
+
+    with open(file_path, "rb") as handle:
+        yield from ijson.items(handle, record_path)
+
+
+def json_to_jsonl(
+    file_path: Union[str, pathlib.Path],
+    output_path: Union[str, pathlib.Path],
+    record_path: str = "item",
+    encoding: str = "utf-8",
+    verbose: bool = False,
+) -> pathlib.Path:
+    """Convert a standard JSON file to JSON Lines.
+
+    Args:
+        file_path: Input JSON file path.
+        output_path: Destination JSON Lines file path.
+        record_path: ijson record path for arrays (default: "item").
+        encoding: Output encoding.
+        verbose: If True, logs progress.
+    """
+    path = coerce_path(file_path, must_exist=True, verbose=verbose, label="file_path")
+    output = coerce_path(
+        output_path, must_exist=False, verbose=verbose, label="output_path"
+    )
+
+    try:
+        iterator = _iter_json_objects(path, record_path)
+    except ImportError:
+        data = json.loads(path.read_text(encoding=encoding))
+        if isinstance(data, list):
+            iterator = iter(data)
+        elif isinstance(data, dict) and record_path in data:
+            iterator = iter(data[record_path])
+        else:
+            raise ValueError(
+                "JSON conversion requires a top-level list or a record_path key."
+            )
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with open(output, "w", encoding=encoding) as handle:
+        count = 0
+        for obj in iterator:
+            handle.write(json.dumps(obj))
+            handle.write("\n")
+            count += 1
+
+    log_info(f"Wrote {count} JSON Lines records to {output}", verbose)
+    return output
+
+
+def _infer_parquet_batch_rows(
+    file_path: pathlib.Path,
+    parquet_file: pq.ParquetFile,
+    memory_fraction: float,
+    verbose: bool,
+) -> int:
+    """Infer an approximate Parquet batch size (rows) to keep memory bounded."""
+    try:
+        available_memory = _get_available_memory()
+        target_memory = int(available_memory * memory_fraction)
+        file_size = file_path.stat().st_size
+        num_rows = int(getattr(parquet_file.metadata, "num_rows", 0) or 0)
+        if num_rows <= 0 or file_size <= 0 or target_memory <= 0:
+            return 65_536
+
+        # Parquet is compressed on disk; materialized batches are larger.
+        # We use a conservative multiplier to avoid overshooting.
+        bytes_per_row_on_disk = file_size / num_rows
+        inflated_bytes_per_row = max(1.0, bytes_per_row_on_disk * 3.0)
+        batch_rows = int(target_memory / inflated_bytes_per_row)
+
+        # Keep within sane bounds.
+        batch_rows = max(1_024, min(1_000_000, batch_rows))
+        log_debug(
+            f"Auto Parquet batch_rows={batch_rows}"
+            f"{format_log_context(build_log_context(file_path=file_path))}",
+            verbose,
+        )
+        return batch_rows
+    except Exception:
+        return 65_536
+
+
+def _get_available_memory() -> int:
+    """Get available system memory in bytes."""
+    try:
+        import psutil
+
+        return psutil.virtual_memory().available
+    except ImportError:
+        log_warning("psutil not installed; assuming 4GB available memory", verbose=True)
+        return 4 * 1024 * 1024 * 1024
+
+
+def _estimate_text_column_count(
+    file_path: pathlib.Path,
+    delimiter: str,
+) -> int:
+    """Estimate the number of columns in a delimited text file."""
+    try:
+        with open(file_path, "r", encoding="utf-8", errors="replace") as handle:
+            header = handle.readline()
+    except Exception:
+        return 1
+
+    if not header:
+        return 1
+
+    return max(1, header.count(delimiter) + 1)
+
+
+def _adaptive_sample_rows(file_size: int, sample_rows: int, column_count: int) -> int:
+    """Adapt sample rows to file size and width to limit costly inference."""
+    rows = max(25, sample_rows)
+
+    if file_size >= _TEXT_MEDIUM_FILE_BYTES:
+        rows = min(rows, 500)
+
+    if column_count >= 300:
+        rows = min(rows, 100)
+    elif column_count >= 100:
+        rows = min(rows, 200)
+
+    return max(25, rows)
+
+
+def _estimate_file_memory(file_path: pathlib.Path, sample_rows: int = 1000) -> int:
+    """
+    Estimate memory required to load a file by sampling.
+
+    Returns estimated bytes needed to load entire file.
+    """
+    file_size = file_path.stat().st_size
+    suffix = file_path.suffix.lower()
+
+    if suffix == ".parquet":
+        return file_size * 3
+
+    if suffix in (".csv", ".tsv", ".txt"):
+        if file_size >= _TEXT_EXTREME_FILE_BYTES:
+            return int(file_size * _TEXT_EXTREME_MULTIPLIER)
+
+        if file_size >= _TEXT_LARGE_FILE_BYTES:
+            return int(file_size * _TEXT_LARGE_MULTIPLIER)
+
+        try:
+            sample_kwargs = {}
+            if suffix == ".tsv":
+                sample_kwargs["sep"] = "\t"
+
+            delimiter = sample_kwargs.get("sep", ",")
+            column_count = _estimate_text_column_count(file_path, delimiter)
+            adaptive_rows = _adaptive_sample_rows(file_size, sample_rows, column_count)
+
+            if file_size >= _TEXT_MEDIUM_FILE_BYTES or column_count >= 200:
+                sample_kwargs["dtype"] = str
+
+            sample = pd.read_csv(file_path, nrows=adaptive_rows, **sample_kwargs)
+            if len(sample) == 0:
+                return file_size * 3
+
+            memory_per_row = sample.memory_usage(deep=True).sum() / len(sample)
+            estimated_rows = _count_lines_estimate(file_path)
+            return int(memory_per_row * estimated_rows * 1.2)
+        except Exception:
+            return file_size * 3
+
+    if suffix in (".xlsx", ".xls"):
+        return file_size * 10
+
+    return file_size * 3
+
+
+def _count_lines_estimate(file_path: pathlib.Path, sample_size: int = 65536) -> int:
+    """Estimate number of lines in a file by sampling."""
+    file_size = file_path.stat().st_size
+    with open(file_path, "rb") as f:
+        sample = f.read(sample_size)
+        lines_in_sample = sample.count(b"\n")
+
+    if lines_in_sample == 0:
+        return 1
+
+    return int(file_size * lines_in_sample / len(sample))
+
+
+def _read_file_chunked_path(
+    path: pathlib.Path,
+    chunksize: Optional[int] = None,
+    memory_fraction: float = 0.5,
+    verbose: bool = False,
+    **read_kwargs,
+) -> Iterator[pd.DataFrame]:
+    """Read a file in chunks from a validated Path."""
+    suffix = path.suffix.lower()
+
+    if suffix == ".parquet":
+        # Parquet can be read in row batches directly from metadata.
+        parquet_file = pq.ParquetFile(path)
+        batch_rows = chunksize
+        if batch_rows is None:
+            batch_rows = _infer_parquet_batch_rows(
+                file_path=path,
+                parquet_file=parquet_file,
+                memory_fraction=memory_fraction,
+                verbose=verbose,
+            )
+
+        for chunk_num, batch in enumerate(
+            parquet_file.iter_batches(batch_size=int(batch_rows), use_threads=True),
+            start=1,
+        ):
+            yield batch.to_pandas()
+            context = format_log_context(
+                build_log_context(
+                    file_path=path,
+                    chunk_index=chunk_num,
+                    chunk_rows=len(batch),
+                )
+            )
+            log_debug(f"Read parquet batch with {len(batch)} rows.{context}", verbose)
+        return
+
+    if suffix not in (".csv", ".tsv", ".txt"):
+        raise ValueError(f"Unsupported file format for chunked reading: {suffix}")
+
+    if chunksize is None:
+        # Auto-size chunks so that each chunk stays under the memory budget.
+        available_memory = _get_available_memory()
+        target_memory = int(available_memory * memory_fraction)
+        estimated_total = _estimate_file_memory(path)
+
+        if estimated_total <= target_memory:
+            context = format_log_context(
+                build_log_context(
+                    file_path=path,
+                    estimated_mb=f"{estimated_total / 1e6:.1f}",
+                    target_mb=f"{target_memory / 1e6:.1f}",
+                )
+            )
+            log_info(f"File fits in memory; reading all at once.{context}", verbose)
+            detected_kwargs = _prepare_text_read_kwargs(
+                file_path=path,
+                suffix=suffix,
+                read_kwargs=read_kwargs,
+                verbose=verbose,
+            )
+            normalized_kwargs = _normalize_text_delimiter_kwargs(
+                suffix, detected_kwargs
+            )
+            df = pd.read_csv(path, **normalized_kwargs)
+            yield df
+            return
+
+        total_lines = _count_lines_estimate(path)
+        memory_per_row = estimated_total / max(1, total_lines)
+        chunksize = max(1000, int(target_memory / memory_per_row))
+        context = format_log_context(
+            build_log_context(
+                file_path=path,
+                estimated_mb=f"{estimated_total / 1e6:.1f}",
+                target_mb=f"{target_memory / 1e6:.1f}",
+                chunk_rows=chunksize,
+            )
+        )
+        log_info(f"File too large; reading in chunks.{context}", verbose)
+
+    chunk_num = 0
+    detected_kwargs = _prepare_text_read_kwargs(
+        file_path=path,
+        suffix=suffix,
+        read_kwargs=read_kwargs,
+        verbose=verbose,
+    )
+    normalized_kwargs = _normalize_text_delimiter_kwargs(suffix, detected_kwargs)
+    for chunk in pd.read_csv(path, chunksize=chunksize, **normalized_kwargs):
+        chunk_num += 1
+        context = format_log_context(
+            build_log_context(file_path=path, chunk_index=chunk_num)
+        )
+        log_debug(f"Read chunk with {len(chunk)} rows.{context}", verbose)
+        yield chunk
+
+
+def read_file_chunked(
+    file_path: Union[str, pathlib.Path],
+    chunksize: Optional[int] = None,
+    memory_fraction: float = 0.5,
+    verbose: bool = False,
+    **read_kwargs,
+) -> Iterator[pd.DataFrame]:
+    """
+    Read a file in chunks, automatically determining chunk size based on available memory.
+
+    Args:
+        file_path: Path to the file to read.
+        chunksize: Optional explicit chunk size (rows). If None, auto-calculated.
+        memory_fraction: Fraction of available memory to use (default: 0.5).
+        verbose: If True, logs progress messages.
+        **read_kwargs: Additional arguments passed to pandas read function.
+
+    Yields:
+        DataFrame chunks.
+
+    Raises:
+        ValueError: If file does not exist or format is unsupported.
+    """
+    path = coerce_path(file_path, must_exist=True, verbose=verbose, label="file_path")
+    yield from _read_file_chunked_path(
+        path=path,
+        chunksize=chunksize,
+        memory_fraction=memory_fraction,
+        verbose=verbose,
+        **read_kwargs,
+    )
+
+
+def read_file_iter(
+    file_path: Union[str, pathlib.Path],
+    chunksize: Optional[int] = None,
+    memory_fraction: float = 0.5,
+    verbose: bool = False,
+    **read_kwargs,
+) -> Iterator[pd.DataFrame]:
+    """Stream a file as an iterator of DataFrame chunks.
+
+    This is the "never materialize" API: unlike read_file_smart(), this function
+    does not concatenate chunks into a single DataFrame.
+
+    Supported streaming formats:
+    - .csv / .tsv / .txt  (via pandas chunking)
+    - .parquet            (via pyarrow iter_batches)
+    - .json               (JSON Lines via pandas chunks; or standard JSON arrays via ijson)
+    - .xlsx / .xls        (via openpyxl read-only streaming, if available)
+
+    Non-streaming formats:
+    - .xlsx / .xls are loaded fully and yielded as a single DataFrame if
+      openpyxl is unavailable and the file is estimated to fit within
+      memory_fraction of available memory.
+
+    Raises:
+        ValueError: If the file is missing, unsupported, or too large for a
+            non-streaming format.
+    """
+    path = coerce_path(file_path, must_exist=True, verbose=verbose, label="file_path")
+
+    suffix = path.suffix.lower()
+
+    if suffix in (".csv", ".tsv", ".txt", ".parquet"):
+        yield from _read_file_chunked_path(
+            path=path,
+            chunksize=chunksize,
+            memory_fraction=memory_fraction,
+            verbose=verbose,
+            **read_kwargs,
+        )
+        return
+
+    if suffix in (".json", ".jsonl"):
+        # pandas can stream JSON only for JSON Lines (one JSON object per line).
+        lines = bool(read_kwargs.get("lines", False))
+        if suffix == ".jsonl" and "lines" not in read_kwargs:
+            read_kwargs = dict(read_kwargs)
+            read_kwargs["lines"] = True
+            lines = True
+        if not lines:
+            record_path = read_kwargs.get("record_path", "item")
+            if chunksize is None:
+                chunksize = 1_000
+                log_info(
+                    "Standard JSON streaming enabled; using default chunksize=1000. "
+                    f"Set record_path='{record_path}' if your JSON is nested.",
+                    verbose,
+                )
+
+            try:
+                iterator = _iter_json_objects(path, record_path)
+            except ImportError:
+                available_memory = _get_available_memory()
+                target_memory = int(available_memory * memory_fraction)
+                estimated_total = _estimate_file_memory(path)
+                if estimated_total > target_memory:
+                    raise ValueError(
+                        "Streaming standard JSON requires the optional 'ijson' dependency. "
+                        "Install ijson or convert the file to JSON Lines with "
+                        "`json_to_jsonl()` (then set lines=True)."
+                    )
+                yield pd.read_json(path, **read_kwargs)
+                return
+
+            buffer: List[dict] = []
+            chunk_num = 0
+            for obj in iterator:
+                buffer.append(obj)
+                if len(buffer) >= int(chunksize):
+                    chunk_num += 1
+                    yield pd.DataFrame(buffer)
+                    log_debug(
+                        f"Read json chunk {chunk_num} with {len(buffer)} rows.",
+                        verbose,
+                    )
+                    buffer = []
+
+            if buffer:
+                chunk_num += 1
+                yield pd.DataFrame(buffer)
+                log_debug(
+                    f"Read json chunk {chunk_num} with {len(buffer)} rows.",
+                    verbose,
+                )
+            return
+
+        # JSON Lines streaming.
+        if chunksize is None:
+            available_memory = _get_available_memory()
+            target_memory = int(available_memory * memory_fraction)
+            estimated_total = _estimate_file_memory(path)
+
+            if estimated_total <= target_memory:
+                yield pd.read_json(path, **read_kwargs)
+                return
+
+            total_lines = _count_lines_estimate(path)
+            memory_per_line = estimated_total / max(1, total_lines)
+            chunksize = max(1000, int(target_memory / max(1.0, memory_per_line)))
+            context = format_log_context(
+                build_log_context(
+                    file_path=path,
+                    estimated_mb=f"{estimated_total / 1e6:.1f}",
+                    target_mb=f"{target_memory / 1e6:.1f}",
+                    chunk_rows=chunksize,
+                )
+            )
+            log_info(f"JSON Lines too large; streaming in chunks.{context}", verbose)
+
+        # pandas returns a TextFileReader-like iterator when chunksize is provided.
+        json_iter = pd.read_json(path, chunksize=chunksize, **read_kwargs)
+        for i, chunk in enumerate(json_iter, start=1):
+            context = format_log_context(
+                build_log_context(file_path=path, chunk_index=i)
+            )
+            log_debug(f"Read json chunk with {len(chunk)} rows.{context}", verbose)
+            yield chunk
+        return
+
+    if suffix in (".xlsx", ".xls"):
+        available_memory = _get_available_memory()
+        target_memory = int(available_memory * memory_fraction)
+        estimated_total = _estimate_file_memory(path)
+        # If the file is estimated too large to load safely and no explicit
+        # chunksize was provided, avoid attempting to open the workbook
+        # (which would error on invalid files) and raise a clear ValueError.
+        if chunksize is None and estimated_total > target_memory:
+            raise ValueError("Excel streaming is not supported for very large files.")
+
+        if chunksize is None and estimated_total <= target_memory:
+            yield pd.read_excel(path, **read_kwargs)
+            return
+
+        try:
+            yield from _read_excel_streaming(
+                file_path=path,
+                chunksize=chunksize,
+                memory_fraction=memory_fraction,
+                verbose=verbose,
+                **read_kwargs,
+            )
+            return
+        except ImportError as exc:
+            if estimated_total > target_memory:
+                raise ValueError(
+                    "Excel streaming is not supported for very large files. "
+                    "Install openpyxl for streaming or convert to CSV/Parquet first."
+                ) from exc
+            yield pd.read_excel(path, **read_kwargs)
+            return
+
+    raise ValueError(f"Unsupported file format for streaming: {suffix}")
+
+
+def _read_excel_streaming(
+    file_path: Union[str, pathlib.Path],
+    chunksize: Optional[int],
+    memory_fraction: float,
+    verbose: bool,
+    **read_kwargs,
+) -> Iterator[pd.DataFrame]:
+    """Stream Excel files using openpyxl read-only mode."""
+    try:
+        import openpyxl
+    except ImportError as exc:  # pragma: no cover - depends on optional dependency
+        raise ImportError("openpyxl is required for Excel streaming") from exc
+
+    path = pathlib.Path(file_path)
+    if chunksize is None:
+        chunksize = 10_000
+        log_info(
+            f"Excel streaming enabled; using default chunksize={chunksize} rows.",
+            verbose,
+        )
+
+    sheet_name = read_kwargs.pop("sheet_name", 0)
+    header = read_kwargs.pop("header", 0)
+    data_only = read_kwargs.pop("data_only", True)
+
+    if isinstance(sheet_name, (list, tuple)):
+        raise ValueError("Excel streaming supports a single sheet_name at a time.")
+
+    @contextmanager
+    def _openpyxl_workbook() -> Iterator[openpyxl.Workbook]:
+        workbook = openpyxl.load_workbook(path, read_only=True, data_only=data_only)
+        try:
+            yield workbook
+        finally:
+            workbook.close()
+
+    with _openpyxl_workbook() as workbook:
+        try:
+            if sheet_name is None:
+                worksheet = workbook.active
+            elif isinstance(sheet_name, int):
+                worksheet = workbook.worksheets[sheet_name]
+            else:
+                worksheet = workbook[sheet_name]
+        except (KeyError, IndexError) as exc:
+            raise ValueError(f"Sheet not found: {sheet_name}") from exc
+
+        row_iter = worksheet.iter_rows(values_only=True)
+        columns: Optional[List[str]] = None
+        if header is not None:
+            header_index = int(header)
+            for _ in range(header_index):
+                next(row_iter, None)
+            header_row = next(row_iter, None)
+            if header_row is None:
+                return
+            columns = ["" if value is None else str(value) for value in header_row]
+
+        buffer: List[List[object]] = []
+        chunk_num = 0
+        for row in row_iter:
+            buffer.append(list(row))
+            if len(buffer) >= int(chunksize):
+                chunk_num += 1
+                yield pd.DataFrame(buffer, columns=columns)
+                log_debug(
+                    f"Read excel chunk {chunk_num} with {len(buffer)} rows.",
+                    verbose,
+                )
+                buffer = []
+
+        if buffer:
+            chunk_num += 1
+            yield pd.DataFrame(buffer, columns=columns)
+            log_debug(f"Read excel chunk {chunk_num} with {len(buffer)} rows.", verbose)
+
+
+def excel_to_parquets(
+    file_path: Union[str, pathlib.Path],
+    output_dir: Union[str, pathlib.Path],
+    output_prefix: str = "part",
+    rows_per_file: Optional[int] = None,
+    memory_fraction: float = 0.5,
+    convert_types: bool = True,
+    verbose: bool = False,
+    **read_kwargs,
+) -> List[pathlib.Path]:
+    """Stream an Excel file to multiple Parquet partitions.
+
+    This requires openpyxl and reads the Excel file in read-only mode.
+    """
+    path = coerce_path(file_path, must_exist=True, verbose=verbose, label="file_path")
+    output_path = ensure_directory(output_dir, verbose=verbose, label="output_dir")
+
+    output_files: List[pathlib.Path] = []
+    part_num = 0
+
+    chunks = _read_excel_streaming(
+        file_path=path,
+        chunksize=rows_per_file,
+        memory_fraction=memory_fraction,
+        verbose=verbose,
+        **read_kwargs,
+    )
+    for chunk in _normalized_chunks(
+        chunks,
+        convert_types=convert_types,
+        verbose=verbose,
+    ):
+        output_file = output_path / f"{output_prefix}_{part_num:05d}.parquet"
+        chunk.to_parquet(output_file, index=False)
+        output_files.append(output_file)
+
+        log_info(f"Wrote {len(chunk)} rows to {output_file}", verbose)
+        part_num += 1
+
+    log_info(f"Created {len(output_files)} Parquet files in {output_path}", verbose)
+    return output_files
+
+
+def read_file_to_parquets(
+    file_path: Union[str, pathlib.Path],
+    output_dir: Union[str, pathlib.Path],
+    output_prefix: str = "part",
+    rows_per_file: Optional[int] = None,
+    memory_fraction: float = 0.5,
+    convert_types: bool = True,
+    verbose: bool = False,
+    **read_kwargs,
+) -> List[pathlib.Path]:
+    """
+    Read a large file and write it to multiple Parquet files if it doesn't fit in memory.
+
+    Args:
+        file_path: Path to the input file.
+        output_dir: Directory where Parquet files will be written.
+        output_prefix: Prefix for output file names (default: "part").
+        rows_per_file: Optional explicit rows per output file. If None, auto-calculated.
+        memory_fraction: Fraction of available memory to use.
+        convert_types: If True, attempts to convert string columns to numeric.
+        verbose: If True, logs progress messages.
+        **read_kwargs: Additional arguments passed to pandas read function.
+
+    Returns:
+        List of paths to the created Parquet files.
+
+    Raises:
+        ValueError: If file does not exist or format is unsupported.
+    """
+    path = coerce_path(file_path, must_exist=True, verbose=verbose, label="file_path")
+    output_path = ensure_directory(output_dir, verbose=verbose, label="output_dir")
+
+    output_files: List[pathlib.Path] = []
+    part_num = 0
+
+    chunks = _read_file_chunked_path(
+        path=path,
+        chunksize=rows_per_file,
+        memory_fraction=memory_fraction,
+        verbose=verbose,
+        **read_kwargs,
+    )
+    for chunk in _normalized_chunks(
+        chunks,
+        convert_types=convert_types,
+        verbose=verbose,
+    ):
+        output_file = output_path / f"{output_prefix}_{part_num:05d}.parquet"
+        chunk.to_parquet(output_file, index=False)
+        output_files.append(output_file)
+
+        log_info(f"Wrote {len(chunk)} rows to {output_file}", verbose)
+        part_num += 1
+
+    log_info(f"Created {len(output_files)} Parquet files in {output_path}", verbose)
+    return output_files
+
+
+def stream_to_sink(
+    chunks: Iterable[pd.DataFrame],
+    output_dir: Union[str, pathlib.Path],
+    output_prefix: str = "part",
+    convert_types: bool = True,
+    verbose: bool = False,
+    sink: Optional[Callable[[pd.DataFrame, pathlib.Path], None]] = None,
+    output_suffix: str = ".parquet",
+) -> List[pathlib.Path]:
+    """Consume an iterator of DataFrames and write incremental partitions.
+
+    Args:
+        chunks: Iterable of DataFrame chunks (e.g., from read_file_iter()).
+        output_dir: Directory where partitions are written.
+        output_prefix: Filename prefix for partitions.
+        convert_types: If True, attempts to convert numeric-looking strings.
+        verbose: If True, logs progress.
+        sink: Optional custom sink function. If omitted, writes Parquet files via
+            DataFrame.to_parquet(output_file).
+        output_suffix: File suffix to use for output files (default: ".parquet").
+
+    Returns:
+        List of output file paths produced by the sink.
+    """
+    output_path = ensure_directory(output_dir, verbose=verbose, label="output_dir")
+
+    output_files: List[pathlib.Path] = []
+    part_num = 0
+
+    if not isinstance(output_suffix, str) or not output_suffix:
+        raise ValueError("output_suffix must be a non-empty string")
+    if not output_suffix.startswith("."):
+        output_suffix = f".{output_suffix}"
+
+    if sink is None:
+
+        def _default_sink(chunk: pd.DataFrame, output_file: pathlib.Path) -> None:
+            chunk.to_parquet(output_file, index=False)
+
+        sink = _default_sink
+
+    with timed_step("stream_to_sink", verbose=verbose):
+        for chunk in _normalized_chunks(
+            chunks,
+            convert_types=convert_types,
+            verbose=verbose,
+        ):
+            output_file = output_path / f"{output_prefix}_{part_num:05d}{output_suffix}"
+            sink(chunk, output_file)
+            output_files.append(output_file)
+
+            log_info(f"Wrote {len(chunk)} rows to {output_file}", verbose)
+            part_num += 1
+
+    log_info(f"Created {len(output_files)} partitions in {output_path}", verbose)
+    return output_files
+
+
+def stream_to_parquets(
+    file_path: Union[str, pathlib.Path],
+    output_dir: Union[str, pathlib.Path],
+    output_prefix: str = "part",
+    rows_per_file: Optional[int] = None,
+    memory_fraction: float = 0.5,
+    convert_types: bool = True,
+    verbose: bool = False,
+    **read_kwargs,
+) -> List[pathlib.Path]:
+    """Stream a file and write it to Parquet partitions without materializing.
+
+    This helper is the "no concat" companion to read_file_to_parquets(). It uses
+    read_file_iter() under the hood and writes each incoming chunk to a separate
+    Parquet file.
+
+    Args:
+        file_path: Input file path.
+        output_dir: Directory where Parquet partitions are written.
+        output_prefix: Output filename prefix.
+        rows_per_file: Desired rows per partition. For streaming formats this
+            is passed as chunksize; if None, chunk sizes are chosen automatically
+            based on memory_fraction.
+        memory_fraction: Fraction of available memory to use when auto-sizing.
+        convert_types: If True, attempts to convert numeric-looking strings.
+        verbose: If True, logs progress.
+        **read_kwargs: Passed to the underlying reader.
+
+    Returns:
+        List of Parquet file paths.
+
+    Raises:
+        ValueError: If the input is missing/unsupported.
+    """
+    chunk_iter = read_file_iter(
+        file_path=file_path,
+        chunksize=rows_per_file,
+        memory_fraction=memory_fraction,
+        verbose=verbose,
+        **read_kwargs,
+    )
+    return stream_to_sink(
+        chunks=chunk_iter,
+        output_dir=output_dir,
+        output_prefix=output_prefix,
+        convert_types=convert_types,
+        verbose=verbose,
+    )
+
+
+def parquet_to_csv_partitions(
+    file_path: Union[str, pathlib.Path],
+    output_dir: Union[str, pathlib.Path],
+    output_prefix: str = "part",
+    rows_per_file: Optional[int] = None,
+    memory_fraction: float = 0.5,
+    convert_types: bool = True,
+    verbose: bool = False,
+    delimiter: str = ",",
+    include_header: bool = True,
+    line_terminator: str = "\n",
+    drop_columns: Optional[list[str]] = None,
+    column_order: Optional[list[str]] = None,
+    drop_extra_columns: bool = False,
+) -> List[pathlib.Path]:
+    """Stream a Parquet file to CSV partitions without materializing.
+
+    Args:
+        file_path: Parquet file path.
+        output_dir: Directory where CSV partitions are written.
+        output_prefix: Output filename prefix.
+        rows_per_file: Desired rows per partition. If None, batch size is chosen
+            automatically based on memory_fraction.
+        memory_fraction: Fraction of available memory to use when auto-sizing.
+        convert_types: If True, attempts to convert numeric-looking strings.
+        verbose: If True, logs progress.
+        delimiter: CSV delimiter.
+        include_header: If True, include headers in each CSV file.
+        line_terminator: Line terminator used in CSV output.
+        drop_columns: Optional column names to drop before writing.
+        column_order: Optional column order to enforce in CSV output.
+        drop_extra_columns: If True, drop columns not in column_order.
+
+    Returns:
+        List of CSV file paths.
+    """
+    path = coerce_path(file_path, must_exist=True, verbose=verbose, label="file_path")
+    if path.suffix.lower() != ".parquet":
+        raise ValueError("file_path must point to a .parquet file")
+
+    chunk_iter = read_file_iter(
+        file_path=path,
+        chunksize=rows_per_file,
+        memory_fraction=memory_fraction,
+        verbose=verbose,
+    )
+
+    drop_set = set(drop_columns or [])
+    if column_order is not None:
+        if not isinstance(column_order, list) or not all(
+            isinstance(col, str) and col.strip() for col in column_order
+        ):
+            raise ValueError("column_order must be a list of non-empty strings")
+        if len(column_order) != len(set(column_order)):
+            raise ValueError("column_order must not contain duplicates")
+
+    def _csv_sink(chunk: pd.DataFrame, output_file: pathlib.Path) -> None:
+        if drop_set:
+            chunk = chunk.drop(columns=[c for c in drop_set if c in chunk.columns])
+        if column_order is not None:
+            missing = [c for c in column_order if c not in chunk.columns]
+            if missing:
+                raise ValueError(
+                    f"Missing columns for CSV output: {missing}. "
+                    "Ensure all Parquet files share the same schema."
+                )
+            if drop_extra_columns:
+                chunk = chunk[[c for c in column_order]]
+            else:
+                extra = [c for c in chunk.columns if c not in column_order]
+                chunk = chunk[[*column_order, *extra]]
+        chunk.to_csv(
+            output_file,
+            index=False,
+            sep=delimiter,
+            header=include_header,
+            lineterminator=line_terminator,
+        )
+
+    return stream_to_sink(
+        chunks=chunk_iter,
+        output_dir=output_dir,
+        output_prefix=output_prefix,
+        convert_types=convert_types,
+        verbose=verbose,
+        sink=_csv_sink,
+        output_suffix=".csv",
+    )
+
+
+def _resolve_return_type(
+    return_polars: bool,
+    return_type: str,
+) -> tuple[str, bool, bool, str]:
+    if return_polars:
+        if return_type not in ("pandas", "polars"):
+            raise ValueError(
+                "return_polars cannot be combined with return_type other than "
+                "'pandas' or 'polars'."
+            )
+        return_type = "polars"
+    normalized_return_type = return_type.lower()
+    wants_polars = normalized_return_type in ("polars", "polars_lazy", "lazy")
+    return_lazy = normalized_return_type in ("polars_lazy", "lazy")
+    return normalized_return_type, wants_polars, return_lazy, return_type
+
+
+def _validate_return_type(normalized_return_type: str, return_type: str) -> None:
+    valid_return_types = {
+        "pandas",
+        "dataframe",
+        "iterator",
+        "polars",
+        "polars_lazy",
+        "lazy",
+    }
+    if normalized_return_type not in valid_return_types:
+        raise ValueError(
+            f"Invalid return_type: {return_type}. Expected one of "
+            f"{sorted(valid_return_types)}."
+        )
+
+
+def _require_polars():
+    try:
+        import polars as pl
+    except ImportError as exc:
+        raise ImportError(
+            "Polars is required for return_type='polars' or 'polars_lazy'."
+        ) from exc
+    return pl
+
+
+def _read_with_polars_return_type(
+    *,
+    path: pathlib.Path,
+    suffix: str,
+    text_read_kwargs: dict,
+    fits_in_memory: bool,
+    return_lazy: bool,
+    verbose: bool,
+    read_pandas_direct: Callable[[], pd.DataFrame],
+    read_pandas_chunked: Callable[[], pd.DataFrame],
+):
+    pl = _require_polars()
+    polars_scan = _polars_scan_source(path, suffix, text_read_kwargs)
+    if polars_scan is not None:
+        if return_lazy:
+            return polars_scan
+        with timed_step("polars_collect", verbose=verbose):
+            df_polars = polars_scan.collect(streaming=not fits_in_memory)
+        context = format_log_context(build_log_context(file_path=path))
+        log_info(f"Polars read {len(df_polars)} rows.{context}", verbose)
+        return df_polars
+
+    if fits_in_memory:
+        with timed_step("pandas_read_direct", verbose=verbose):
+            df_pandas = read_pandas_direct()
+    else:
+        with timed_step("pandas_read_chunked", verbose=verbose):
+            df_pandas = read_pandas_chunked()
+    if return_lazy:
+        return pl.from_pandas(df_pandas).lazy()
+    return pl.from_pandas(df_pandas)
+
+
+def _read_with_polars_fallback(
+    *,
+    path: pathlib.Path,
+    suffix: str,
+    text_read_kwargs: dict,
+    fits_in_memory: bool,
+    use_polars: bool,
+    estimated_memory: int,
+    target_memory: int,
+    verbose: bool,
+) -> Optional[pd.DataFrame]:
+    if not use_polars or fits_in_memory:
+        return None
+    try:
+        context = format_log_context(
+            build_log_context(
+                file_path=path,
+                estimated_mb=f"{estimated_memory / 1e6:.1f}",
+                target_mb=f"{target_memory / 1e6:.1f}",
+            )
+        )
+        log_info(f"Using Polars for large file.{context}", verbose)
+
+        lf = _polars_scan_source(path, suffix, text_read_kwargs)
+        if lf is None:
+            raise ValueError(f"Unsupported format for Polars: {suffix}")
+
+        with timed_step("polars_collect", verbose=verbose):
+            df_polars = lf.collect(streaming=True)
+        context = format_log_context(build_log_context(file_path=path))
+        log_info(f"Polars read {len(df_polars)} rows.{context}", verbose)
+        return df_polars.to_pandas()
+
+    except ImportError:
+        context = format_log_context(build_log_context(file_path=path))
+        log_warning(
+            "Polars not installed; falling back to pandas chunked reading."
+            f"{context}",
+            verbose,
+        )
+    except Exception as e:
+        context = format_log_context(build_log_context(file_path=path))
+        log_warning(f"Polars failed: {e}; falling back to pandas.{context}", verbose)
+    return None
+
+
+def read_file_smart(
+    file_path: Union[str, pathlib.Path],
+    use_polars: bool = True,
+    return_polars: bool = False,
+    return_type: str = "pandas",
+    memory_fraction: float = 0.5,
+    verbose: bool = False,
+    **read_kwargs,
+) -> Union[pd.DataFrame, "pl.DataFrame", "pl.LazyFrame"]:
+    """
+    Intelligently read a file, using Polars for large files if available.
+
+    For files that fit in memory, reads directly. For large files, uses
+    Polars lazy evaluation or pandas chunking as a fallback.
+
+    Args:
+        file_path: Path to the file to read.
+        use_polars: If True and Polars is available, uses Polars for large files.
+        return_polars: If True, return a Polars DataFrame (alias for return_type="polars").
+        return_type: "pandas" (default), "polars", "polars_lazy", or "iterator".
+        memory_fraction: Fraction of available memory to use.
+        verbose: If True, logs progress messages.
+        return_type: One of "pandas", "polars", "polars_lazy", or "iterator".
+        **read_kwargs: Additional arguments passed to the read function.
+
+    Returns:
+        DataFrame with the file contents, or an iterator when return_type="iterator".
+
+    Raises:
+        ValueError: If file does not exist or format is unsupported.
+    """
+    (
+        normalized_return_type,
+        wants_polars,
+        return_lazy,
+        return_type,
+    ) = _resolve_return_type(return_polars, return_type)
+
+    path = coerce_path(file_path, must_exist=True, verbose=verbose, label="file_path")
+
+    suffix = path.suffix.lower()
+    if suffix == ".jsonl" and "lines" not in read_kwargs:
+        read_kwargs = dict(read_kwargs)
+        read_kwargs["lines"] = True
+
+    if normalized_return_type == "iterator":
+        return read_file_iter(
+            file_path=path,
+            memory_fraction=memory_fraction,
+            verbose=verbose,
+            **read_kwargs,
+        )
+    available_memory = _get_available_memory()
+    target_memory = int(available_memory * memory_fraction)
+    estimated_memory = _estimate_file_memory(path)
+
+    fits_in_memory = estimated_memory <= target_memory
+    _validate_return_type(normalized_return_type, return_type)
+
+    context = format_log_context(
+        build_log_context(
+            file_path=path,
+            estimated_mb=f"{estimated_memory / 1e6:.1f}",
+            target_mb=f"{target_memory / 1e6:.1f}",
+            fits=fits_in_memory,
+        )
+    )
+    log_debug(f"Computed file memory fit.{context}", verbose)
+
+    text_read_kwargs = (
+        _prepare_text_read_kwargs(
+            file_path=path,
+            suffix=suffix,
+            read_kwargs=read_kwargs,
+            verbose=verbose,
+        )
+        if suffix in (".csv", ".tsv", ".txt")
+        else dict(read_kwargs)
+    )
+
+    def _read_pandas_direct() -> pd.DataFrame:
+        if suffix == ".parquet":
+            return pd.read_parquet(path, **read_kwargs)
+        if suffix in (".csv", ".tsv", ".txt"):
+            normalized_kwargs = _normalize_text_delimiter_kwargs(
+                suffix, text_read_kwargs
+            )
+            return pd.read_csv(path, **normalized_kwargs)
+        if suffix in (".xlsx", ".xls"):
+            return pd.read_excel(path, **read_kwargs)
+        if suffix in (".json", ".jsonl"):
+            return pd.read_json(path, **read_kwargs)
+        raise ValueError(f"Unsupported file format: {suffix}")
+
+    def _read_pandas_chunked() -> pd.DataFrame:
+        normalized_kwargs = _normalize_text_delimiter_kwargs(suffix, text_read_kwargs)
+        return pd.concat(
+            _read_file_chunked_path(
+                path=path,
+                memory_fraction=memory_fraction,
+                verbose=verbose,
+                **normalized_kwargs,
+            ),
+            ignore_index=True,
+        )
+
+    def _read_pandas_with_logging() -> pd.DataFrame:
+        if fits_in_memory:
+            context = format_log_context(
+                build_log_context(
+                    file_path=path,
+                    estimated_mb=f"{estimated_memory / 1e6:.1f}",
+                    target_mb=f"{target_memory / 1e6:.1f}",
+                )
+            )
+            log_info(f"Reading file directly.{context}", verbose)
+            with timed_step("pandas_read_direct", verbose=verbose):
+                return _read_pandas_direct()
+
+        context = format_log_context(
+            build_log_context(
+                file_path=path,
+                estimated_mb=f"{estimated_memory / 1e6:.1f}",
+                target_mb=f"{target_memory / 1e6:.1f}",
+            )
+        )
+        log_info(f"Reading large file in chunks.{context}", verbose)
+        with timed_step("pandas_read_chunked", verbose=verbose):
+            return _read_pandas_chunked()
+
+    if wants_polars:
+        return _read_with_polars_return_type(
+            path=path,
+            suffix=suffix,
+            text_read_kwargs=text_read_kwargs,
+            fits_in_memory=fits_in_memory,
+            return_lazy=return_lazy,
+            verbose=verbose,
+            read_pandas_direct=_read_pandas_direct,
+            read_pandas_chunked=_read_pandas_chunked,
+        )
+
+    polars_df = _read_with_polars_fallback(
+        path=path,
+        suffix=suffix,
+        text_read_kwargs=text_read_kwargs,
+        fits_in_memory=fits_in_memory,
+        use_polars=use_polars,
+        estimated_memory=estimated_memory,
+        target_memory=target_memory,
+        verbose=verbose,
+    )
+    if polars_df is not None:
+        return polars_df
+
+    return _read_pandas_with_logging()