truthound 1.0.8__py3-none-any.whl

truthound/validators/completeness/__init__.py

@@ -0,0 +1,39 @@
+"""Completeness validators for null and missing value detection."""
+
+from truthound.validators.completeness.null import (
+    NullValidator,
+    NotNullValidator,
+    CompletenessRatioValidator,
+)
+from truthound.validators.completeness.empty import (
+    EmptyStringValidator,
+    WhitespaceOnlyValidator,
+)
+from truthound.validators.completeness.conditional import ConditionalNullValidator
+from truthound.validators.completeness.default import DefaultValueValidator
+from truthound.validators.completeness.nan import (
+    NaNValidator,
+    NotNaNValidator,
+    NaNRatioValidator,
+    InfinityValidator,
+    FiniteValidator,
+)
+
+__all__ = [
+    # Null validators
+    "NullValidator",
+    "NotNullValidator",
+    "CompletenessRatioValidator",
+    # Empty string validators
+    "EmptyStringValidator",
+    "WhitespaceOnlyValidator",
+    # Conditional validators
+    "ConditionalNullValidator",
+    "DefaultValueValidator",
+    # NaN validators (float-specific)
+    "NaNValidator",
+    "NotNaNValidator",
+    "NaNRatioValidator",
+    "InfinityValidator",
+    "FiniteValidator",
+]

truthound/validators/completeness/conditional.py

@@ -0,0 +1,73 @@
+"""Conditional null validators."""
+
+from typing import Any, Callable
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class ConditionalNullValidator(Validator):
+    """Validates null values based on conditions in other columns.
+
+    Example:
+        # Email should not be null when subscription_type is 'newsletter'
+        validator = ConditionalNullValidator(
+            column="email",
+            condition_column="subscription_type",
+            condition_values=["newsletter"],
+        )
+    """
+
+    name = "conditional_null"
+    category = "completeness"
+
+    def __init__(
+        self,
+        column: str,
+        condition_column: str,
+        condition_values: list[Any],
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column = column
+        self.condition_column = condition_column
+        self.condition_values = condition_values
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Filter rows where condition is met
+        filtered = lf.filter(
+            pl.col(self.condition_column).is_in(self.condition_values)
+        )
+
+        result = filtered.select([
+            pl.len().alias("_total"),
+            pl.col(self.column).null_count().alias("_null"),
+        ]).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        null_count = result["_null"][0]
+        if null_count > 0:
+            null_pct = null_count / total_rows
+            condition_str = f"{self.condition_column} in {self.condition_values}"
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="conditional_null",
+                    count=null_count,
+                    severity=Severity.HIGH,
+                    details=f"Found {null_count} nulls when {condition_str}",
+                    expected=0,
+                    actual=null_count,
+                )
+            )
+
+        return issues

truthound/validators/completeness/default.py

@@ -0,0 +1,98 @@
+"""Default value detection validators."""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class DefaultValueValidator(Validator):
+    """Detects excessive use of default/placeholder values.
+
+    Example:
+        validator = DefaultValueValidator(
+            default_values=["N/A", "TBD", "unknown", -1, 0],
+            max_ratio=0.1,
+        )
+    """
+
+    name = "default_value"
+    category = "completeness"
+
+    # Common default/placeholder values
+    COMMON_DEFAULTS = [
+        "N/A", "n/a", "NA", "na",
+        "TBD", "tbd",
+        "unknown", "Unknown", "UNKNOWN",
+        "none", "None", "NONE",
+        "-", "--", "---",
+        "null", "NULL",
+        "undefined",
+        0, -1, -999, 9999, 99999,
+    ]
+
+    def __init__(
+        self,
+        default_values: list[Any] | None = None,
+        max_ratio: float = 0.1,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.default_values = default_values or self.COMMON_DEFAULTS
+        self.max_ratio = max_ratio
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_target_columns(lf)
+
+        if not columns:
+            return issues
+
+        # Separate default values by type to avoid is_in type mismatch
+        string_defaults = [v for v in self.default_values if isinstance(v, str)]
+        numeric_defaults = [v for v in self.default_values if isinstance(v, (int, float)) and not isinstance(v, bool)]
+
+        df = lf.collect()
+        total_rows = len(df)
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            col_data = df.get_column(col)
+            dtype = col_data.dtype
+
+            # Choose appropriate defaults based on column type
+            if dtype in (pl.Utf8, pl.String):
+                defaults_to_check = string_defaults
+            elif dtype.is_numeric():
+                defaults_to_check = numeric_defaults
+            else:
+                # For other types, skip or check string representation
+                continue
+
+            if not defaults_to_check:
+                continue
+
+            default_count = col_data.is_in(defaults_to_check).sum()
+
+            if default_count > 0:
+                ratio = default_count / total_rows
+                if ratio > self.max_ratio:
+                    issues.append(
+                        ValidationIssue(
+                            column=col,
+                            issue_type="default_value",
+                            count=default_count,
+                            severity=self._calculate_severity(ratio),
+                            details=f"{ratio:.1%} values are defaults (max {self.max_ratio:.1%})",
+                            expected=f"<= {self.max_ratio:.1%}",
+                            actual=f"{ratio:.1%}",
+                        )
+                    )
+
+        return issues

truthound/validators/completeness/empty.py

@@ -0,0 +1,103 @@
+"""Empty string validators."""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import (
+    ValidationIssue,
+    Validator,
+    StringValidatorMixin,
+)
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class EmptyStringValidator(Validator, StringValidatorMixin):
+    """Detects empty strings in columns."""
+
+    name = "empty_string"
+    category = "completeness"
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_string_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.len().alias("_total"),
+            *[
+                (pl.col(c) == "").sum().alias(f"_empty_{c}")
+                for c in columns
+            ],
+        ]
+        result = lf.select(exprs).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            empty_count = result[f"_empty_{col}"][0]
+            if empty_count > 0:
+                empty_pct = empty_count / total_rows
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="empty_string",
+                        count=empty_count,
+                        severity=self._calculate_severity(empty_pct),
+                        details=f"{empty_pct:.1%} of values are empty strings",
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class WhitespaceOnlyValidator(Validator, StringValidatorMixin):
+    """Detects strings that contain only whitespace."""
+
+    name = "whitespace_only"
+    category = "completeness"
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_string_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.len().alias("_total"),
+            *[
+                (
+                    (pl.col(c).str.strip_chars() == "") & pl.col(c).is_not_null()
+                ).sum().alias(f"_ws_{c}")
+                for c in columns
+            ],
+        ]
+        result = lf.select(exprs).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            ws_count = result[f"_ws_{col}"][0]
+            if ws_count > 0:
+                ws_pct = ws_count / total_rows
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="whitespace_only",
+                        count=ws_count,
+                        severity=self._calculate_severity(ws_pct),
+                        details=f"{ws_pct:.1%} of values contain only whitespace",
+                    )
+                )
+
+        return issues

truthound/validators/completeness/nan.py

@@ -0,0 +1,337 @@
+"""NaN (Not a Number) validators for float columns.
+
+NaN values are distinct from NULL/None in Polars:
+- NULL (None): Missing value, works with all types
+- NaN: IEEE 754 floating-point "Not a Number", only for Float32/Float64
+
+This module provides validators specifically for detecting NaN values,
+which are often introduced by:
+- Division by zero (0/0)
+- Invalid math operations (sqrt(-1))
+- Data parsing errors
+- External data sources
+"""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import (
+    ValidationIssue,
+    Validator,
+    FloatValidatorMixin,
+)
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class NaNValidator(Validator, FloatValidatorMixin):
+    """Detects NaN (Not a Number) values in float columns.
+
+    Unlike NullValidator which detects None/missing values, this validator
+    specifically finds IEEE 754 NaN values in Float32/Float64 columns.
+
+    Example:
+        validator = NaNValidator()
+        issues = validator.validate(lf)
+
+        # Only check specific columns
+        validator = NaNValidator(columns=["price", "ratio"])
+
+        # With severity override
+        validator = NaNValidator(severity_override=Severity.CRITICAL)
+    """
+
+    name = "nan"
+    category = "completeness"
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_float_columns(lf)
+
+        if not columns:
+            return issues
+
+        # Single optimized query for all float columns
+        exprs = [
+            pl.len().alias("_total"),
+            *[pl.col(c).is_nan().sum().alias(f"_nan_{c}") for c in columns],
+            *[pl.col(c).count().alias(f"_count_{c}") for c in columns],
+        ]
+        result = lf.select(exprs).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            nan_count = result[f"_nan_{col}"][0]
+            non_null_count = result[f"_count_{col}"][0]
+
+            if nan_count > 0:
+                # Calculate ratio against non-null values
+                ratio = nan_count / non_null_count if non_null_count > 0 else 1.0
+
+                # Check mostly threshold
+                if self._passes_mostly(nan_count, non_null_count):
+                    continue
+
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="nan",
+                        count=nan_count,
+                        severity=self._calculate_severity(ratio),
+                        details=f"{ratio:.1%} of non-null values are NaN",
+                        expected="No NaN values",
+                        actual=f"{nan_count} NaN values",
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class NotNaNValidator(Validator, FloatValidatorMixin):
+    """Validates that float columns contain no NaN values.
+
+    Similar to NotNullValidator but for NaN values specifically.
+    Reports HIGH severity by default since NaN often indicates data corruption.
+
+    Example:
+        validator = NotNaNValidator(columns=["price"])
+        issues = validator.validate(lf)
+    """
+
+    name = "not_nan"
+    category = "completeness"
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_float_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.len().alias("_total"),
+            *[pl.col(c).is_nan().sum().alias(f"_nan_{c}") for c in columns],
+        ]
+        result = lf.select(exprs).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            nan_count = result[f"_nan_{col}"][0]
+
+            if nan_count > 0:
+                nan_pct = nan_count / total_rows
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="not_nan_violation",
+                        count=nan_count,
+                        severity=Severity.HIGH,
+                        details=f"Expected no NaN, found {nan_count} ({nan_pct:.1%})",
+                        expected=0,
+                        actual=nan_count,
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class NaNRatioValidator(Validator, FloatValidatorMixin):
+    """Validates that NaN ratio is below a threshold.
+
+    Useful when some NaN values are acceptable but need to be controlled.
+
+    Example:
+        # Allow up to 5% NaN values
+        validator = NaNRatioValidator(max_ratio=0.05)
+
+        # Check only specific columns
+        validator = NaNRatioValidator(columns=["score"], max_ratio=0.01)
+    """
+
+    name = "nan_ratio"
+    category = "completeness"
+
+    def __init__(
+        self,
+        max_ratio: float = 0.0,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        if not 0.0 <= max_ratio <= 1.0:
+            raise ValueError(f"max_ratio must be between 0 and 1, got {max_ratio}")
+        self.max_ratio = max_ratio
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_float_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.len().alias("_total"),
+            *[pl.col(c).is_nan().sum().alias(f"_nan_{c}") for c in columns],
+            *[pl.col(c).count().alias(f"_count_{c}") for c in columns],
+        ]
+        result = lf.select(exprs).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            nan_count = result[f"_nan_{col}"][0]
+            non_null_count = result[f"_count_{col}"][0]
+
+            if non_null_count == 0:
+                continue
+
+            ratio = nan_count / non_null_count
+
+            if ratio > self.max_ratio:
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="nan_ratio_exceeded",
+                        count=nan_count,
+                        severity=self._calculate_severity(
+                            ratio - self.max_ratio,
+                            thresholds=(0.3, 0.1, 0.01),
+                        ),
+                        details=f"NaN ratio {ratio:.1%} > max {self.max_ratio:.1%}",
+                        expected=f"<= {self.max_ratio:.1%}",
+                        actual=f"{ratio:.1%}",
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class InfinityValidator(Validator, FloatValidatorMixin):
+    """Detects infinity values (inf, -inf) in float columns.
+
+    Infinity values can cause issues in calculations and aggregations.
+    This validator detects both positive and negative infinity.
+
+    Example:
+        validator = InfinityValidator()
+        issues = validator.validate(lf)
+    """
+
+    name = "infinity"
+    category = "completeness"
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_float_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.len().alias("_total"),
+            *[pl.col(c).is_infinite().sum().alias(f"_inf_{c}") for c in columns],
+            *[pl.col(c).count().alias(f"_count_{c}") for c in columns],
+        ]
+        result = lf.select(exprs).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            inf_count = result[f"_inf_{col}"][0]
+            non_null_count = result[f"_count_{col}"][0]
+
+            if inf_count > 0:
+                ratio = inf_count / non_null_count if non_null_count > 0 else 1.0
+
+                if self._passes_mostly(inf_count, non_null_count):
+                    continue
+
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="infinity",
+                        count=inf_count,
+                        severity=self._calculate_severity(ratio),
+                        details=f"{inf_count} infinite values detected",
+                        expected="No infinity values",
+                        actual=f"{inf_count} inf values ({ratio:.1%})",
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class FiniteValidator(Validator, FloatValidatorMixin):
+    """Validates that float columns contain only finite values.
+
+    Checks for both NaN and infinity values at once. This is equivalent
+    to running both NaNValidator and InfinityValidator but more efficient.
+
+    Example:
+        validator = FiniteValidator(columns=["price", "quantity"])
+        issues = validator.validate(lf)
+    """
+
+    name = "finite"
+    category = "completeness"
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_float_columns(lf)
+
+        if not columns:
+            return issues
+
+        # Check for non-finite values (NaN or Inf)
+        exprs = [
+            pl.len().alias("_total"),
+            *[
+                (pl.col(c).is_nan() | pl.col(c).is_infinite()).sum().alias(f"_nonfinite_{c}")
+                for c in columns
+            ],
+            *[pl.col(c).count().alias(f"_count_{c}") for c in columns],
+        ]
+        result = lf.select(exprs).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            nonfinite_count = result[f"_nonfinite_{col}"][0]
+            non_null_count = result[f"_count_{col}"][0]
+
+            if nonfinite_count > 0:
+                ratio = nonfinite_count / non_null_count if non_null_count > 0 else 1.0
+
+                if self._passes_mostly(nonfinite_count, non_null_count):
+                    continue
+
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="not_finite",
+                        count=nonfinite_count,
+                        severity=Severity.HIGH,
+                        details=f"{nonfinite_count} non-finite values (NaN or Inf)",
+                        expected="All finite values",
+                        actual=f"{nonfinite_count} non-finite ({ratio:.1%})",
+                    )
+                )
+
+        return issues