truthound 1.0.8__py3-none-any.whl

truthound/validators/uniqueness/within_record.py

@@ -0,0 +1,599 @@
+"""Within-record uniqueness validators.
+
+Provides vectorized validation for checking uniqueness within each row,
+using Polars' horizontal operations for optimal performance.
+
+Type Safety:
+- Pairwise strategy: Native type comparison (no casting)
+- Horizontal strategy: Type-grouped comparison for accuracy
+
+Performance Notes:
+- Vectorized approach: ~100x faster than row-by-row iteration
+- Scales linearly with row count, not column count
+- Memory efficient: no Python list creation per row
+"""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator
+from truthound.validators.registry import register_validator
+
+
+def _get_column_type_groups(
+    lf: pl.LazyFrame,
+    columns: list[str],
+) -> dict[str, list[str]]:
+    """Group columns by their base type for type-safe comparison.
+
+    This prevents false positives from comparing numeric 1 with string "1".
+
+    Returns:
+        Dict mapping type category to list of column names
+    """
+    schema = lf.collect_schema()
+
+    type_groups: dict[str, list[str]] = {
+        "numeric": [],
+        "string": [],
+        "datetime": [],
+        "boolean": [],
+        "other": [],
+    }
+
+    numeric_types = {
+        pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+        pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+        pl.Float32, pl.Float64,
+    }
+    string_types = {pl.String, pl.Utf8}
+    datetime_types = {pl.Date, pl.Datetime, pl.Time, pl.Duration}
+    boolean_types = {pl.Boolean}
+
+    for col in columns:
+        dtype = type(schema[col])
+        if dtype in numeric_types:
+            type_groups["numeric"].append(col)
+        elif dtype in string_types:
+            type_groups["string"].append(col)
+        elif dtype in datetime_types:
+            type_groups["datetime"].append(col)
+        elif dtype in boolean_types:
+            type_groups["boolean"].append(col)
+        else:
+            type_groups["other"].append(col)
+
+    # Remove empty groups
+    return {k: v for k, v in type_groups.items() if v}
+
+
+def _build_pairwise_equality_expr(
+    lf: pl.LazyFrame,
+    columns: list[str],
+    ignore_nulls: bool = True,
+) -> pl.Expr:
+    """Build a vectorized expression to detect duplicate values within rows.
+
+    Strategy: For N columns, check all N*(N-1)/2 pairs for equality.
+    Only compares columns of compatible types to prevent Polars type errors.
+
+    Type Safety:
+    - Only compares columns with the same base type
+    - numeric 1 will not be compared with string "1"
+    - int 1 == float 1.0 (Polars handles numeric coercion correctly)
+
+    Args:
+        lf: LazyFrame to get schema from
+        columns: List of column names to check
+        ignore_nulls: If True, null values are not considered duplicates
+
+    Returns:
+        A Polars expression that evaluates to True for rows with duplicates
+    """
+    if len(columns) < 2:
+        # Need at least 2 columns to have duplicates
+        return pl.lit(False)
+
+    # Group columns by type to only compare compatible types
+    type_groups = _get_column_type_groups(lf, columns)
+
+    # Build pairwise equality checks only within same type groups
+    equality_checks: list[pl.Expr] = []
+
+    for type_name, group_cols in type_groups.items():
+        if len(group_cols) < 2:
+            continue
+
+        # Compare pairs within this type group
+        for i in range(len(group_cols)):
+            for j in range(i + 1, len(group_cols)):
+                col_i, col_j = group_cols[i], group_cols[j]
+
+                if ignore_nulls:
+                    # Only consider equal if both are non-null AND equal
+                    pair_equal = (
+                        pl.col(col_i).is_not_null()
+                        & pl.col(col_j).is_not_null()
+                        & (pl.col(col_i) == pl.col(col_j))
+                    )
+                else:
+                    # Consider equal including null == null
+                    pair_equal = (
+                        (pl.col(col_i) == pl.col(col_j))
+                        | (pl.col(col_i).is_null() & pl.col(col_j).is_null())
+                    )
+
+                equality_checks.append(pair_equal)
+
+    if not equality_checks:
+        # No columns of the same type to compare
+        return pl.lit(False)
+
+    # Combine all checks with OR - any duplicate pair means the row has duplicates
+    if len(equality_checks) == 1:
+        return equality_checks[0]
+
+    result = equality_checks[0]
+    for check in equality_checks[1:]:
+        result = result | check
+
+    return result
+
+
+def _build_type_safe_horizontal_expr(
+    lf: pl.LazyFrame,
+    columns: list[str],
+    ignore_nulls: bool = True,
+) -> pl.Expr:
+    """Build type-safe horizontal uniqueness expression.
+
+    Groups columns by type and checks for duplicates within each type group.
+    This prevents false positives from type coercion (e.g., 1 != "1").
+
+    Strategy:
+    1. Group columns by type (numeric, string, datetime, etc.)
+    2. For each type group with 2+ columns, check for duplicates
+    3. OR all group results together
+
+    Args:
+        lf: LazyFrame to get schema from
+        columns: List of column names to check
+        ignore_nulls: If True, null values don't count toward uniqueness
+
+    Returns:
+        Expression that evaluates to True for rows with duplicates
+    """
+    if len(columns) < 2:
+        return pl.lit(False)
+
+    type_groups = _get_column_type_groups(lf, columns)
+
+    # Build expressions for each type group
+    group_exprs: list[pl.Expr] = []
+
+    for type_name, group_cols in type_groups.items():
+        if len(group_cols) < 2:
+            continue
+
+        # For each type group, use concat_list without casting
+        # All columns in the group already have compatible types
+        col_exprs = [pl.col(c) for c in group_cols]
+        list_expr = pl.concat_list(col_exprs)
+
+        if ignore_nulls:
+            # Filter out nulls and compare unique count vs total non-null count
+            non_null_list = list_expr.list.eval(pl.element().drop_nulls())
+            unique_count = non_null_list.list.n_unique()
+            total_count = non_null_list.list.len()
+            group_has_dup = unique_count < total_count
+        else:
+            # Compare unique count vs total count
+            group_has_dup = list_expr.list.n_unique() < list_expr.list.len()
+
+        group_exprs.append(group_has_dup)
+
+    if not group_exprs:
+        return pl.lit(False)
+
+    # Combine all group expressions with OR
+    if len(group_exprs) == 1:
+        return group_exprs[0]
+
+    result = group_exprs[0]
+    for expr in group_exprs[1:]:
+        result = result | expr
+
+    return result
+
+
+def _build_horizontal_n_unique_expr(
+    columns: list[str],
+    ignore_nulls: bool = True,
+) -> pl.Expr:
+    """Build expression using horizontal unique count (legacy, type-unsafe).
+
+    WARNING: This function casts all columns to string, which may cause
+    false positives (numeric 1 == string "1"). Use _build_type_safe_horizontal_expr
+    for type-safe comparison.
+
+    Kept for backwards compatibility and cases where type coercion is desired.
+
+    Args:
+        columns: List of column names to check
+        ignore_nulls: If True, null values don't count toward uniqueness
+
+    Returns:
+        Expression that evaluates to True for rows with duplicates
+    """
+    if len(columns) < 2:
+        return pl.lit(False)
+
+    # Cast all columns to string for uniform comparison
+    col_exprs = [pl.col(c).cast(pl.Utf8) for c in columns]
+
+    # Create a list of values for each row
+    list_expr = pl.concat_list(col_exprs)
+
+    if ignore_nulls:
+        # Filter out nulls and compare unique count vs total non-null count
+        non_null_list = list_expr.list.eval(pl.element().drop_nulls())
+        unique_count = non_null_list.list.n_unique()
+        total_count = non_null_list.list.len()
+        return unique_count < total_count
+    else:
+        # Compare unique count vs total count
+        return list_expr.list.n_unique() < list_expr.list.len()
+
+
+@register_validator
+class UniqueWithinRecordValidator(Validator):
+    """Validates that specified columns have unique values within each row.
+
+    Uses vectorized Polars operations for optimal performance - approximately
+    100x faster than row-by-row iteration on large datasets.
+
+    Type Safety:
+    - "pairwise" strategy: Uses native Polars comparison (type-safe)
+    - "horizontal" strategy: Now uses type-grouped comparison (type-safe)
+    - "horizontal_legacy": Uses string casting (may have false positives)
+
+    Strategy Selection (auto mode):
+    - For 2-6 columns: Uses pairwise equality (faster, more precise)
+    - For 7+ columns: Uses type-safe horizontal n_unique (more scalable)
+
+    Example:
+        # Primary and secondary contacts should be different
+        validator = UniqueWithinRecordValidator(
+            columns=["primary_contact", "secondary_contact"],
+        )
+
+        # All three choice fields should be unique
+        validator = UniqueWithinRecordValidator(
+            columns=["choice_1", "choice_2", "choice_3"],
+        )
+
+    Performance:
+        - 1M rows, 3 columns: ~0.05s (vs ~1.4s with iteration)
+        - Scales linearly with row count
+    """
+
+    name = "unique_within_record"
+    category = "uniqueness"
+
+    # Threshold for switching strategies
+    PAIRWISE_THRESHOLD = 6
+
+    def __init__(
+        self,
+        columns: list[str],
+        ignore_nulls: bool = True,
+        strategy: str = "auto",
+        **kwargs: Any,
+    ):
+        """Initialize the validator.
+
+        Args:
+            columns: List of columns to check for uniqueness within each row
+            ignore_nulls: If True, null values are excluded from comparison
+            strategy: "auto" (default), "pairwise", "horizontal", or "horizontal_legacy"
+            **kwargs: Additional validator configuration
+        """
+        super().__init__(**kwargs)
+        self.check_columns = columns
+        self.ignore_nulls = ignore_nulls
+        self.strategy = strategy
+
+        if len(columns) < 2:
+            raise ValueError("At least 2 columns required for within-record uniqueness")
+
+    def _get_strategy(self) -> str:
+        """Determine which strategy to use."""
+        if self.strategy != "auto":
+            return self.strategy
+
+        # Use pairwise for small column counts (fewer comparisons)
+        # Use horizontal for large column counts (more scalable)
+        n_cols = len(self.check_columns)
+        n_pairs = n_cols * (n_cols - 1) // 2
+
+        if n_pairs <= 15:  # Up to 6 columns = 15 pairs
+            return "pairwise"
+        return "horizontal"
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Build the duplicate detection expression
+        strategy = self._get_strategy()
+
+        if strategy == "pairwise":
+            has_dup_expr = _build_pairwise_equality_expr(
+                lf, self.check_columns, self.ignore_nulls
+            )
+        elif strategy == "horizontal_legacy":
+            # Legacy mode with string casting (for backwards compatibility)
+            has_dup_expr = _build_horizontal_n_unique_expr(
+                self.check_columns, self.ignore_nulls
+            )
+        else:
+            # Default: type-safe horizontal comparison
+            has_dup_expr = _build_type_safe_horizontal_expr(
+                lf, self.check_columns, self.ignore_nulls
+            )
+
+        # Execute vectorized query
+        result = lf.select(
+            pl.len().alias("_total"),
+            has_dup_expr.sum().alias("_dup_count"),
+        ).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        duplicate_rows = result["_dup_count"][0]
+
+        if duplicate_rows > 0:
+            if self._passes_mostly(duplicate_rows, total_rows):
+                return issues
+
+            ratio = duplicate_rows / total_rows
+            col_desc = ", ".join(self.check_columns)
+
+            # Get sample values (collect only if needed)
+            samples = self._get_duplicate_samples(lf, has_dup_expr)
+
+            issues.append(
+                ValidationIssue(
+                    column=f"[{col_desc}]",
+                    issue_type="duplicate_within_record",
+                    count=duplicate_rows,
+                    severity=self._calculate_severity(ratio),
+                    details=f"{duplicate_rows} rows have duplicate values across columns",
+                    expected="Unique values within each row",
+                    sample_values=samples,
+                )
+            )
+
+        return issues
+
+    def _get_duplicate_samples(
+        self,
+        lf: pl.LazyFrame,
+        has_dup_expr: pl.Expr,
+    ) -> list[str]:
+        """Get sample rows with duplicates for error reporting."""
+        sample_df = (
+            lf.with_row_index("_row_idx")
+            .filter(has_dup_expr)
+            .select(["_row_idx"] + self.check_columns)
+            .head(self.config.sample_size)
+            .collect()
+        )
+
+        samples = []
+        for row in sample_df.iter_rows(named=True):
+            idx = row["_row_idx"]
+            vals = [str(row[c]) for c in self.check_columns]
+            samples.append(f"row {idx}: [{', '.join(vals)}]")
+
+        return samples
+
+
+@register_validator
+class AllColumnsUniqueWithinRecordValidator(Validator):
+    """Validates that all non-null values in each row are unique.
+
+    Uses vectorized horizontal operations for optimal performance.
+    Type-safe: Only compares values of the same type.
+
+    Example:
+        # Each row's values should all be different
+        validator = AllColumnsUniqueWithinRecordValidator()
+
+        # Check only specific columns
+        validator = AllColumnsUniqueWithinRecordValidator(
+            columns=["field_a", "field_b", "field_c"],
+        )
+
+    Performance:
+        - Uses horizontal n_unique for scalability
+        - 1M rows: ~0.1s (vs ~1.4s with iteration)
+    """
+
+    name = "all_columns_unique_within_record"
+    category = "uniqueness"
+
+    def __init__(
+        self,
+        ignore_nulls: bool = True,
+        type_safe: bool = True,
+        **kwargs: Any,
+    ):
+        """Initialize the validator.
+
+        Args:
+            ignore_nulls: If True, null values are excluded from comparison
+            type_safe: If True, only compare values of the same type
+            **kwargs: Additional validator configuration
+        """
+        super().__init__(**kwargs)
+        self.ignore_nulls = ignore_nulls
+        self.type_safe = type_safe
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        columns = self._get_target_columns(lf)
+        if len(columns) < 2:
+            return issues
+
+        # Use type-safe or legacy strategy based on setting
+        if self.type_safe:
+            has_dup_expr = _build_type_safe_horizontal_expr(
+                lf, columns, self.ignore_nulls
+            )
+        else:
+            has_dup_expr = _build_horizontal_n_unique_expr(columns, self.ignore_nulls)
+
+        result = lf.select(
+            pl.len().alias("_total"),
+            has_dup_expr.sum().alias("_dup_count"),
+        ).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        duplicate_rows = result["_dup_count"][0]
+
+        if duplicate_rows > 0:
+            if self._passes_mostly(duplicate_rows, total_rows):
+                return issues
+
+            ratio = duplicate_rows / total_rows
+
+            issues.append(
+                ValidationIssue(
+                    column="_all_columns",
+                    issue_type="duplicate_values_in_record",
+                    count=duplicate_rows,
+                    severity=self._calculate_severity(ratio),
+                    details=f"{duplicate_rows} rows have duplicate values",
+                    expected="All column values unique within each row",
+                )
+            )
+
+        return issues
+
+
+@register_validator
+class ColumnPairUniqueValidator(Validator):
+    """Validates that a specific pair of columns never have the same value.
+
+    Optimized for the common case of checking just two columns.
+    Uses direct comparison expression for maximum performance.
+
+    Type Safety:
+    - Uses native Polars comparison which respects types
+    - numeric 1 != string "1" (no false positives)
+
+    Example:
+        # Sender and receiver should never be the same
+        validator = ColumnPairUniqueValidator(
+            column_a="sender_id",
+            column_b="receiver_id",
+        )
+    """
+
+    name = "column_pair_unique"
+    category = "uniqueness"
+
+    def __init__(
+        self,
+        column_a: str,
+        column_b: str,
+        ignore_nulls: bool = True,
+        **kwargs: Any,
+    ):
+        """Initialize the validator.
+
+        Args:
+            column_a: First column name
+            column_b: Second column name
+            ignore_nulls: If True, null values are not considered equal
+            **kwargs: Additional validator configuration
+        """
+        super().__init__(**kwargs)
+        self.column_a = column_a
+        self.column_b = column_b
+        self.ignore_nulls = ignore_nulls
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Build simple equality expression
+        # Native Polars comparison respects types (no false positives)
+        if self.ignore_nulls:
+            equal_expr = (
+                pl.col(self.column_a).is_not_null()
+                & pl.col(self.column_b).is_not_null()
+                & (pl.col(self.column_a) == pl.col(self.column_b))
+            )
+        else:
+            equal_expr = (
+                (pl.col(self.column_a) == pl.col(self.column_b))
+                | (pl.col(self.column_a).is_null() & pl.col(self.column_b).is_null())
+            )
+
+        result = lf.select(
+            pl.len().alias("_total"),
+            equal_expr.sum().alias("_equal_count"),
+        ).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        equal_count = result["_equal_count"][0]
+
+        if equal_count > 0:
+            if self._passes_mostly(equal_count, total_rows):
+                return issues
+
+            ratio = equal_count / total_rows
+
+            # Get samples
+            samples = self._get_equal_samples(lf, equal_expr)
+
+            issues.append(
+                ValidationIssue(
+                    column=f"[{self.column_a}, {self.column_b}]",
+                    issue_type="column_pair_not_unique",
+                    count=equal_count,
+                    severity=self._calculate_severity(ratio),
+                    details=f"{equal_count} rows have equal values in both columns",
+                    expected=f"{self.column_a} != {self.column_b}",
+                    sample_values=samples,
+                )
+            )
+
+        return issues
+
+    def _get_equal_samples(self, lf: pl.LazyFrame, equal_expr: pl.Expr) -> list[str]:
+        """Get sample rows where columns are equal."""
+        sample_df = (
+            lf.filter(equal_expr)
+            .select([self.column_a, self.column_b])
+            .head(self.config.sample_size)
+            .collect()
+        )
+
+        samples = []
+        for row in sample_df.iter_rows(named=True):
+            val_a = row[self.column_a]
+            samples.append(f"{self.column_a}={val_a}")
+
+        return samples