truthound 1.0.8__py3-none-any.whl

truthound/validators/streaming/__init__.py

@@ -0,0 +1,126 @@
+"""Streaming validators for memory-efficient large dataset processing.
+
+This module provides validators optimized for processing very large datasets
+that don't fit in memory. Key features:
+
+1. Chunked Processing: Process data in configurable chunks
+2. Result Aggregation: Combine results across chunks accurately
+3. Memory Bounded: Predictable memory usage regardless of data size
+4. Compatible: Works with standard LazyFrame/DataFrame inputs
+5. File-Based Streaming: Direct streaming from Parquet, CSV, Arrow IPC files
+6. Arrow Flight Streaming: Distributed data streaming from remote servers
+
+Usage:
+    from truthound.validators.streaming import (
+        StreamingNullValidator,
+        StreamingRangeValidator,
+        StreamingValidationPipeline,
+        # New streaming sources
+        ParquetStreamingSource,
+        CSVStreamingSource,
+        ArrowIPCStreamingSource,
+        ArrowFlightStreamingSource,
+        # Convenience functions
+        stream_validate,
+        stream_validate_many,
+    )
+
+    # Single streaming validator
+    validator = StreamingNullValidator(chunk_size=50_000)
+    issues = validator.validate(very_large_lazyframe)
+
+    # Pipeline of streaming validators
+    pipeline = StreamingValidationPipeline(
+        validators=[
+            StreamingNullValidator(),
+            StreamingRangeValidator(min_value=0),
+        ],
+        chunk_size=100_000,
+    )
+    issues = pipeline.validate(very_large_lazyframe)
+
+    # File-based streaming (new)
+    with ParquetStreamingSource("huge_file.parquet", chunk_size=100_000) as source:
+        for chunk_df in source:
+            issues = validator.validate(chunk_df.lazy())
+
+    # One-liner streaming validation (new)
+    issues = stream_validate(validator, "huge_file.parquet")
+
+    # Multi-validator streaming (new)
+    results = stream_validate_many(
+        [validator1, validator2],
+        "huge_file.parquet",
+    )
+"""
+
+from truthound.validators.streaming.base import (
+    StreamingValidator,
+    StreamingValidationPipeline,
+    StreamingState,
+    ChunkResult,
+)
+from truthound.validators.streaming.completeness import (
+    StreamingNullValidator,
+    StreamingCompletenessValidator,
+)
+from truthound.validators.streaming.range import (
+    StreamingRangeValidator,
+    StreamingOutlierValidator,
+)
+
+# New streaming sources
+from truthound.validators.streaming.sources import (
+    StreamingSource,
+    StreamingSourceConfig,
+    ParquetStreamingSource,
+    CSVStreamingSource,
+    JSONLStreamingSource,
+    ArrowIPCStreamingSource,
+    ArrowFlightStreamingSource,
+    LazyFrameStreamingSource,
+    create_streaming_source,
+)
+
+# New streaming mixin and utilities
+from truthound.validators.streaming.mixin import (
+    StreamingValidatorMixin,
+    StreamingValidatorAdapter,
+    StreamingAccumulator,
+    CountingAccumulator,
+    SamplingAccumulator,
+    stream_validate,
+    stream_validate_many,
+)
+
+__all__ = [
+    # Base classes
+    "StreamingValidator",
+    "StreamingValidationPipeline",
+    "StreamingState",
+    "ChunkResult",
+    # Completeness validators
+    "StreamingNullValidator",
+    "StreamingCompletenessValidator",
+    # Range validators
+    "StreamingRangeValidator",
+    "StreamingOutlierValidator",
+    # Streaming sources
+    "StreamingSource",
+    "StreamingSourceConfig",
+    "ParquetStreamingSource",
+    "CSVStreamingSource",
+    "JSONLStreamingSource",
+    "ArrowIPCStreamingSource",
+    "ArrowFlightStreamingSource",
+    "LazyFrameStreamingSource",
+    "create_streaming_source",
+    # Streaming mixin and utilities
+    "StreamingValidatorMixin",
+    "StreamingValidatorAdapter",
+    "StreamingAccumulator",
+    "CountingAccumulator",
+    "SamplingAccumulator",
+    "stream_validate",
+    "stream_validate_many",
+]

truthound/validators/streaming/base.py

@@ -0,0 +1,292 @@
+"""Base classes for streaming validators.
+
+Provides abstract base classes and utilities for implementing validators
+that process data in chunks to support very large datasets.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Iterator, Generic, TypeVar
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import (
+    ValidationIssue,
+    Validator,
+    ValidatorConfig,
+)
+
+
+T = TypeVar("T")
+
+
+@dataclass
+class ChunkResult(Generic[T]):
+    """Result from processing a single chunk.
+
+    Attributes:
+        chunk_index: Index of the processed chunk
+        row_offset: Starting row offset in the original dataset
+        row_count: Number of rows in this chunk
+        data: Chunk-specific aggregated data (e.g., counts, sums)
+        issues: Issues found in this chunk (before aggregation)
+    """
+
+    chunk_index: int
+    row_offset: int
+    row_count: int
+    data: T
+    issues: list[ValidationIssue] = field(default_factory=list)
+
+
+@dataclass
+class StreamingState:
+    """State maintained across chunks during streaming validation.
+
+    Attributes:
+        total_rows: Total rows processed so far
+        chunk_count: Number of chunks processed
+        column_stats: Per-column accumulated statistics
+    """
+
+    total_rows: int = 0
+    chunk_count: int = 0
+    column_stats: dict[str, dict[str, Any]] = field(default_factory=dict)
+
+    def update_column_stat(
+        self,
+        column: str,
+        key: str,
+        value: Any,
+        aggregator: str = "sum",
+    ) -> None:
+        """Update a column statistic with aggregation.
+
+        Args:
+            column: Column name
+            key: Statistic key
+            value: Value to aggregate
+            aggregator: Aggregation method ('sum', 'max', 'min', 'last')
+        """
+        if column not in self.column_stats:
+            self.column_stats[column] = {}
+
+        if key not in self.column_stats[column]:
+            self.column_stats[column][key] = value
+        else:
+            current = self.column_stats[column][key]
+            if aggregator == "sum":
+                self.column_stats[column][key] = current + value
+            elif aggregator == "max":
+                self.column_stats[column][key] = max(current, value)
+            elif aggregator == "min":
+                self.column_stats[column][key] = min(current, value)
+            elif aggregator == "last":
+                self.column_stats[column][key] = value
+
+    def get_column_stat(self, column: str, key: str, default: Any = None) -> Any:
+        """Get a column statistic."""
+        return self.column_stats.get(column, {}).get(key, default)
+
+
+class StreamingValidator(Validator, ABC):
+    """Abstract base class for streaming validators.
+
+    Streaming validators process data in chunks, maintaining state across
+    chunks and aggregating results at the end. This enables validation of
+    datasets larger than available memory.
+
+    Subclasses must implement:
+    - process_chunk(): Process a single chunk and update state
+    - finalize(): Generate final issues from accumulated state
+
+    Example:
+        class MyStreamingValidator(StreamingValidator):
+            def process_chunk(self, chunk_df, state):
+                # Count something in this chunk
+                count = len(chunk_df.filter(...))
+                state.update_column_stat("col", "count", count, "sum")
+
+            def finalize(self, state, total_rows):
+                issues = []
+                count = state.get_column_stat("col", "count", 0)
+                if count > threshold:
+                    issues.append(ValidationIssue(...))
+                return issues
+    """
+
+    name = "streaming_base"
+    category = "streaming"
+
+    default_chunk_size: int = 100_000
+
+    def __init__(
+        self,
+        chunk_size: int | None = None,
+        config: ValidatorConfig | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(config, **kwargs)
+        self.chunk_size = chunk_size or self.default_chunk_size
+
+    def _iter_chunks(
+        self,
+        lf: pl.LazyFrame,
+        total_rows: int,
+    ) -> Iterator[tuple[int, int, pl.DataFrame]]:
+        """Iterate over chunks of the LazyFrame.
+
+        Yields:
+            Tuples of (chunk_index, row_offset, chunk_dataframe)
+        """
+        chunk_index = 0
+        for offset in range(0, total_rows, self.chunk_size):
+            chunk_df = lf.slice(offset, self.chunk_size).collect()
+            yield chunk_index, offset, chunk_df
+            chunk_index += 1
+
+    @abstractmethod
+    def process_chunk(
+        self,
+        chunk_df: pl.DataFrame,
+        state: StreamingState,
+        chunk_index: int,
+        row_offset: int,
+    ) -> None:
+        """Process a single chunk and update state.
+
+        Args:
+            chunk_df: The chunk DataFrame to process
+            state: Streaming state to update
+            chunk_index: Index of current chunk
+            row_offset: Row offset of this chunk in original data
+        """
+        pass
+
+    @abstractmethod
+    def finalize(
+        self,
+        state: StreamingState,
+        total_rows: int,
+    ) -> list[ValidationIssue]:
+        """Generate final validation issues from accumulated state.
+
+        Args:
+            state: Final streaming state after all chunks
+            total_rows: Total rows in the dataset
+
+        Returns:
+            List of validation issues
+        """
+        pass
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        """Run streaming validation on the LazyFrame.
+
+        Processes data in chunks to limit memory usage.
+
+        Args:
+            lf: LazyFrame to validate
+
+        Returns:
+            List of aggregated validation issues
+        """
+        # Get total rows
+        total_rows = lf.select(pl.len()).collect().item()
+
+        if total_rows == 0:
+            return []
+
+        # Initialize state
+        state = StreamingState()
+
+        # Process chunks
+        for chunk_index, row_offset, chunk_df in self._iter_chunks(lf, total_rows):
+            self.process_chunk(chunk_df, state, chunk_index, row_offset)
+            state.chunk_count += 1
+            state.total_rows += len(chunk_df)
+
+        # Generate final issues
+        return self.finalize(state, total_rows)
+
+
+class StreamingValidationPipeline:
+    """Pipeline for running multiple streaming validators efficiently.
+
+    Processes all validators in a single pass through the data chunks,
+    sharing the chunking overhead across validators.
+
+    Example:
+        pipeline = StreamingValidationPipeline(
+            validators=[
+                StreamingNullValidator(),
+                StreamingRangeValidator(min_value=0, max_value=100),
+            ],
+            chunk_size=50_000,
+        )
+        all_issues = pipeline.validate(large_lazyframe)
+    """
+
+    def __init__(
+        self,
+        validators: list[StreamingValidator],
+        chunk_size: int = 100_000,
+    ):
+        self.validators = validators
+        self.chunk_size = chunk_size
+
+        # Override chunk size for all validators
+        for v in self.validators:
+            v.chunk_size = chunk_size
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        """Run all validators in a single pass through the data.
+
+        Args:
+            lf: LazyFrame to validate
+
+        Returns:
+            Combined list of issues from all validators
+        """
+        total_rows = lf.select(pl.len()).collect().item()
+
+        if total_rows == 0:
+            return []
+
+        # Initialize state for each validator
+        states = [StreamingState() for _ in self.validators]
+
+        # Single pass through chunks
+        chunk_index = 0
+        for offset in range(0, total_rows, self.chunk_size):
+            chunk_df = lf.slice(offset, self.chunk_size).collect()
+
+            # Process chunk with each validator
+            for validator, state in zip(self.validators, states):
+                validator.process_chunk(chunk_df, state, chunk_index, offset)
+                state.chunk_count += 1
+                state.total_rows += len(chunk_df)
+
+            chunk_index += 1
+
+        # Finalize all validators
+        all_issues: list[ValidationIssue] = []
+        for validator, state in zip(self.validators, states):
+            issues = validator.finalize(state, total_rows)
+            all_issues.extend(issues)
+
+        return all_issues
+
+    def add_validator(self, validator: StreamingValidator) -> "StreamingValidationPipeline":
+        """Add a validator to the pipeline.
+
+        Args:
+            validator: Streaming validator to add
+
+        Returns:
+            Self for chaining
+        """
+        validator.chunk_size = self.chunk_size
+        self.validators.append(validator)
+        return self

truthound/validators/streaming/completeness.py

@@ -0,0 +1,210 @@
+"""Streaming completeness validators for large datasets."""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue
+from truthound.validators.streaming.base import StreamingValidator, StreamingState
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class StreamingNullValidator(StreamingValidator):
+    """Streaming version of NullValidator for very large datasets.
+
+    Processes data in chunks and aggregates null counts across chunks.
+
+    Example:
+        validator = StreamingNullValidator(chunk_size=100_000)
+        issues = validator.validate(large_lazyframe)
+    """
+
+    name = "streaming_null"
+    category = "streaming"
+
+    def process_chunk(
+        self,
+        chunk_df: pl.DataFrame,
+        state: StreamingState,
+        chunk_index: int,
+        row_offset: int,
+    ) -> None:
+        """Count nulls in this chunk."""
+        columns = self._get_target_columns(chunk_df.lazy())
+
+        for col in columns:
+            null_count = chunk_df.get_column(col).null_count()
+            total_count = len(chunk_df)
+
+            state.update_column_stat(col, "null_count", null_count, "sum")
+            state.update_column_stat(col, "total_count", total_count, "sum")
+
+    def finalize(
+        self,
+        state: StreamingState,
+        total_rows: int,
+    ) -> list[ValidationIssue]:
+        """Generate issues from aggregated null counts."""
+        issues: list[ValidationIssue] = []
+
+        for col, stats in state.column_stats.items():
+            null_count = stats.get("null_count", 0)
+
+            if null_count > 0:
+                null_pct = null_count / total_rows if total_rows > 0 else 0
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="null",
+                        count=null_count,
+                        severity=self._calculate_severity(null_pct),
+                        details=f"{null_pct:.1%} of values are null (streaming)",
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class StreamingCompletenessValidator(StreamingValidator):
+    """Streaming completeness ratio validator.
+
+    Validates that columns meet minimum completeness ratio.
+
+    Example:
+        validator = StreamingCompletenessValidator(
+            min_ratio=0.95,
+            chunk_size=50_000,
+        )
+    """
+
+    name = "streaming_completeness"
+    category = "streaming"
+
+    def __init__(
+        self,
+        min_ratio: float = 0.95,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.min_ratio = min_ratio
+
+    def process_chunk(
+        self,
+        chunk_df: pl.DataFrame,
+        state: StreamingState,
+        chunk_index: int,
+        row_offset: int,
+    ) -> None:
+        """Count non-null values in this chunk."""
+        columns = self._get_target_columns(chunk_df.lazy())
+
+        for col in columns:
+            non_null_count = chunk_df.get_column(col).drop_nulls().len()
+            total_count = len(chunk_df)
+
+            state.update_column_stat(col, "non_null_count", non_null_count, "sum")
+            state.update_column_stat(col, "total_count", total_count, "sum")
+
+    def finalize(
+        self,
+        state: StreamingState,
+        total_rows: int,
+    ) -> list[ValidationIssue]:
+        """Check completeness ratio against threshold."""
+        issues: list[ValidationIssue] = []
+
+        for col, stats in state.column_stats.items():
+            non_null_count = stats.get("non_null_count", 0)
+            total_count = stats.get("total_count", total_rows)
+
+            if total_count == 0:
+                continue
+
+            ratio = non_null_count / total_count
+
+            if ratio < self.min_ratio:
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="completeness_ratio",
+                        count=total_count - non_null_count,
+                        severity=self._calculate_severity(
+                            1 - ratio, thresholds=(0.5, 0.2, 0.05)
+                        ),
+                        details=f"Completeness {ratio:.1%} < {self.min_ratio:.1%} (streaming)",
+                        expected=self.min_ratio,
+                        actual=ratio,
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class StreamingNaNValidator(StreamingValidator):
+    """Streaming NaN detector for large float datasets.
+
+    Processes data in chunks to detect NaN values with bounded memory.
+
+    Example:
+        validator = StreamingNaNValidator(chunk_size=100_000)
+        issues = validator.validate(large_float_data)
+    """
+
+    name = "streaming_nan"
+    category = "streaming"
+
+    def process_chunk(
+        self,
+        chunk_df: pl.DataFrame,
+        state: StreamingState,
+        chunk_index: int,
+        row_offset: int,
+    ) -> None:
+        """Count NaN values in float columns."""
+        schema = chunk_df.schema
+
+        for col, dtype in schema.items():
+            if dtype in (pl.Float32, pl.Float64):
+                # Check config filters
+                if self.config.columns and col not in self.config.columns:
+                    continue
+                if self.config.exclude_columns and col in self.config.exclude_columns:
+                    continue
+
+                nan_count = chunk_df.get_column(col).is_nan().sum()
+                non_null_count = chunk_df.get_column(col).drop_nulls().len()
+
+                state.update_column_stat(col, "nan_count", nan_count, "sum")
+                state.update_column_stat(col, "non_null_count", non_null_count, "sum")
+
+    def finalize(
+        self,
+        state: StreamingState,
+        total_rows: int,
+    ) -> list[ValidationIssue]:
+        """Generate issues from aggregated NaN counts."""
+        issues: list[ValidationIssue] = []
+
+        for col, stats in state.column_stats.items():
+            nan_count = stats.get("nan_count", 0)
+            non_null_count = stats.get("non_null_count", 0)
+
+            if nan_count > 0:
+                ratio = nan_count / non_null_count if non_null_count > 0 else 1.0
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="nan",
+                        count=nan_count,
+                        severity=self._calculate_severity(ratio),
+                        details=f"{ratio:.1%} of non-null values are NaN (streaming)",
+                        expected="No NaN values",
+                        actual=f"{nan_count} NaN values",
+                    )
+                )
+
+        return issues