truthound 1.0.8__py3-none-any.whl

truthound/validators/multi_column/consistency.py

@@ -0,0 +1,312 @@
+"""Multi-column consistency validators.
+
+Validators for checking logical consistency across multiple columns.
+"""
+
+from typing import Any, Callable
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator
+from truthound.validators.multi_column.base import MultiColumnValidator
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class ColumnConsistencyValidator(Validator):
+    """Validates logical consistency between columns using rules.
+
+    Example:
+        # If status is 'shipped', ship_date must not be null
+        validator = ColumnConsistencyValidator(
+            rules=[
+                {
+                    "when": pl.col("status") == "shipped",
+                    "then": pl.col("ship_date").is_not_null(),
+                    "description": "Shipped orders need ship date",
+                },
+                {
+                    "when": pl.col("status") == "cancelled",
+                    "then": pl.col("cancel_reason").is_not_null(),
+                    "description": "Cancelled orders need reason",
+                },
+            ]
+        )
+    """
+
+    name = "column_consistency"
+    category = "multi_column"
+
+    def __init__(
+        self,
+        rules: list[dict[str, Any]],
+        fail_on_first: bool = False,
+        **kwargs: Any,
+    ):
+        """Initialize consistency validator.
+
+        Args:
+            rules: List of rule dicts with 'when', 'then', and optional 'description'
+            fail_on_first: Stop after first failing rule
+            **kwargs: Additional config
+        """
+        super().__init__(**kwargs)
+        self.rules = rules
+        self.fail_on_first = fail_on_first
+
+        # Validate rules structure
+        for i, rule in enumerate(rules):
+            if "when" not in rule or "then" not in rule:
+                raise ValueError(f"Rule {i} must have 'when' and 'then' keys")
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        for i, rule in enumerate(self.rules):
+            when_expr = rule["when"]
+            then_expr = rule["then"]
+            description = rule.get("description", f"Rule {i}")
+
+            # Count rows where condition is true but then is false
+            result = lf.select([
+                pl.len().alias("_total"),
+                when_expr.sum().alias("_when_true"),
+                (when_expr & ~then_expr).sum().alias("_failures"),
+            ]).collect()
+
+            when_true = result["_when_true"][0]
+            failures = result["_failures"][0]
+
+            if failures > 0:
+                if self._passes_mostly(failures, when_true):
+                    continue
+
+                ratio = failures / when_true if when_true > 0 else 0
+
+                issues.append(
+                    ValidationIssue(
+                        column="_consistency",
+                        issue_type="consistency_rule_failed",
+                        count=failures,
+                        severity=self._calculate_severity(ratio),
+                        details=f"Rule '{description}' failed: {failures}/{when_true} rows",
+                        expected=description,
+                    )
+                )
+
+                if self.fail_on_first:
+                    break
+
+        return issues
+
+
+@register_validator
+class ColumnMutualExclusivityValidator(MultiColumnValidator):
+    """Validates that columns are mutually exclusive (only one can have value).
+
+    Example:
+        # Only one payment method should be set
+        validator = ColumnMutualExclusivityValidator(
+            columns=["credit_card", "bank_transfer", "paypal"],
+            allow_none=True,  # Allow none to be set
+        )
+    """
+
+    name = "column_mutual_exclusivity"
+    category = "multi_column"
+
+    def __init__(
+        self,
+        columns: list[str],
+        allow_none: bool = False,
+        allow_multiple: int = 1,
+        **kwargs: Any,
+    ):
+        """Initialize mutual exclusivity validator.
+
+        Args:
+            columns: Columns to check
+            allow_none: Allow all columns to be null/empty
+            allow_multiple: Number of columns that can have values (default 1)
+            **kwargs: Additional config
+        """
+        super().__init__(columns=columns, **kwargs)
+        self.allow_none = allow_none
+        self.allow_multiple = allow_multiple
+
+    def get_validation_expression(self) -> pl.Expr:
+        """Return mutual exclusivity expression."""
+        # Count non-null columns
+        non_null_count = pl.lit(0)
+        for col in self.columns:
+            non_null_count = non_null_count + pl.col(col).is_not_null().cast(pl.Int32)
+
+        if self.allow_none:
+            return non_null_count <= self.allow_multiple
+        else:
+            return (non_null_count >= 1) & (non_null_count <= self.allow_multiple)
+
+    def _get_issue_type(self) -> str:
+        return "mutual_exclusivity_violated"
+
+    def _get_expected(self) -> str:
+        if self.allow_multiple == 1:
+            if self.allow_none:
+                return f"At most one of {self.columns} should have value"
+            return f"Exactly one of {self.columns} should have value"
+        return f"At most {self.allow_multiple} of {self.columns} should have values"
+
+
+@register_validator
+class ColumnCoexistenceValidator(MultiColumnValidator):
+    """Validates that columns either all have values or all are null.
+
+    Example:
+        # Address fields should all be filled or all empty
+        validator = ColumnCoexistenceValidator(
+            columns=["street", "city", "zip_code", "country"],
+        )
+    """
+
+    name = "column_coexistence"
+    category = "multi_column"
+
+    def get_validation_expression(self) -> pl.Expr:
+        """Return coexistence expression."""
+        # All null or all non-null
+        null_exprs = [pl.col(c).is_null() for c in self.columns]
+        non_null_exprs = [pl.col(c).is_not_null() for c in self.columns]
+
+        all_null = null_exprs[0]
+        all_non_null = non_null_exprs[0]
+
+        for i in range(1, len(self.columns)):
+            all_null = all_null & null_exprs[i]
+            all_non_null = all_non_null & non_null_exprs[i]
+
+        return all_null | all_non_null
+
+    def _get_issue_type(self) -> str:
+        return "coexistence_violated"
+
+    def _get_expected(self) -> str:
+        return f"Columns {self.columns} should all have values or all be null"
+
+
+@register_validator
+class ColumnDependencyValidator(Validator):
+    """Validates that certain columns are filled when a condition is met.
+
+    Example:
+        # When type is 'subscription', billing fields are required
+        validator = ColumnDependencyValidator(
+            condition_column="type",
+            condition_value="subscription",
+            required_columns=["billing_cycle", "next_billing_date"],
+        )
+    """
+
+    name = "column_dependency"
+    category = "multi_column"
+
+    def __init__(
+        self,
+        condition_column: str,
+        condition_value: Any | list[Any],
+        required_columns: list[str],
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.condition_column = condition_column
+        self.condition_values = (
+            condition_value if isinstance(condition_value, list) else [condition_value]
+        )
+        self.required_columns = required_columns
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Build condition expression
+        condition = pl.col(self.condition_column).is_in(self.condition_values)
+
+        # Check each required column
+        for req_col in self.required_columns:
+            result = lf.select([
+                condition.sum().alias("_condition_true"),
+                (condition & pl.col(req_col).is_null()).sum().alias("_missing"),
+            ]).collect()
+
+            condition_true = result["_condition_true"][0]
+            missing = result["_missing"][0]
+
+            if missing > 0:
+                if self._passes_mostly(missing, condition_true):
+                    continue
+
+                ratio = missing / condition_true if condition_true > 0 else 0
+
+                issues.append(
+                    ValidationIssue(
+                        column=req_col,
+                        issue_type="column_dependency_violated",
+                        count=missing,
+                        severity=self._calculate_severity(ratio),
+                        details=f"Column '{req_col}' is null when {self.condition_column} in {self.condition_values}",
+                        expected=f"'{req_col}' required when {self.condition_column} in {self.condition_values}",
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class ColumnImplicationValidator(MultiColumnValidator):
+    """Validates implication: if column A has value X, column B must have value Y.
+
+    Example:
+        # If country is 'US', currency must be 'USD'
+        validator = ColumnImplicationValidator(
+            antecedent_column="country",
+            antecedent_value="US",
+            consequent_column="currency",
+            consequent_value="USD",
+        )
+    """
+
+    name = "column_implication"
+    category = "multi_column"
+
+    def __init__(
+        self,
+        antecedent_column: str,
+        antecedent_value: Any | list[Any],
+        consequent_column: str,
+        consequent_value: Any | list[Any],
+        **kwargs: Any,
+    ):
+        super().__init__(columns=[antecedent_column, consequent_column], **kwargs)
+        self.antecedent_column = antecedent_column
+        self.antecedent_values = (
+            antecedent_value if isinstance(antecedent_value, list) else [antecedent_value]
+        )
+        self.consequent_column = consequent_column
+        self.consequent_values = (
+            consequent_value if isinstance(consequent_value, list) else [consequent_value]
+        )
+
+    def get_validation_expression(self) -> pl.Expr:
+        """Return implication expression (A -> B is equivalent to ~A | B)."""
+        antecedent = pl.col(self.antecedent_column).is_in(self.antecedent_values)
+        consequent = pl.col(self.consequent_column).is_in(self.consequent_values)
+
+        return ~antecedent | consequent
+
+    def _get_issue_type(self) -> str:
+        return "column_implication_violated"
+
+    def _get_expected(self) -> str:
+        return (
+            f"If {self.antecedent_column} in {self.antecedent_values}, "
+            f"then {self.consequent_column} must be in {self.consequent_values}"
+        )

truthound/validators/multi_column/statistical.py

@@ -0,0 +1,299 @@
+"""Multi-column statistical validators.
+
+Validators for checking statistical relationships between columns.
+"""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class ColumnCorrelationValidator(Validator):
+    """Validates correlation between two numeric columns.
+
+    Example:
+        # Height and weight should be positively correlated
+        validator = ColumnCorrelationValidator(
+            column_a="height",
+            column_b="weight",
+            min_correlation=0.3,
+        )
+
+        # No strong correlation between independent variables
+        validator = ColumnCorrelationValidator(
+            column_a="feature_1",
+            column_b="feature_2",
+            max_correlation=0.7,
+        )
+    """
+
+    name = "column_correlation"
+    category = "multi_column"
+
+    def __init__(
+        self,
+        column_a: str,
+        column_b: str,
+        min_correlation: float | None = None,
+        max_correlation: float | None = None,
+        expected_correlation: float | None = None,
+        tolerance: float = 0.1,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column_a = column_a
+        self.column_b = column_b
+        self.min_correlation = min_correlation
+        self.max_correlation = max_correlation
+        self.expected_correlation = expected_correlation
+        self.tolerance = tolerance
+
+        if min_correlation is None and max_correlation is None and expected_correlation is None:
+            raise ValueError(
+                "At least one of 'min_correlation', 'max_correlation', or 'expected_correlation' required"
+            )
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Calculate Pearson correlation
+        df = lf.select([self.column_a, self.column_b]).drop_nulls().collect()
+
+        if len(df) < 3:
+            issues.append(
+                ValidationIssue(
+                    column=f"{self.column_a}, {self.column_b}",
+                    issue_type="insufficient_data_for_correlation",
+                    count=len(df),
+                    severity=Severity.MEDIUM,
+                    details="Need at least 3 data points for correlation",
+                    expected=">= 3 data points",
+                )
+            )
+            return issues
+
+        # Compute correlation using Polars
+        correlation = df.select(
+            pl.corr(self.column_a, self.column_b).alias("corr")
+        )["corr"][0]
+
+        if correlation is None:
+            issues.append(
+                ValidationIssue(
+                    column=f"{self.column_a}, {self.column_b}",
+                    issue_type="correlation_computation_failed",
+                    count=1,
+                    severity=Severity.HIGH,
+                    details="Could not compute correlation (possibly constant values)",
+                    expected="Computable correlation",
+                )
+            )
+            return issues
+
+        # Check bounds
+        if self.expected_correlation is not None:
+            if abs(correlation - self.expected_correlation) > self.tolerance:
+                issues.append(
+                    ValidationIssue(
+                        column=f"{self.column_a}, {self.column_b}",
+                        issue_type="correlation_mismatch",
+                        count=1,
+                        severity=Severity.MEDIUM,
+                        details=f"Correlation {correlation:.3f} differs from expected {self.expected_correlation}",
+                        expected=f"Correlation ≈ {self.expected_correlation} ± {self.tolerance}",
+                    )
+                )
+
+        if self.min_correlation is not None and correlation < self.min_correlation:
+            issues.append(
+                ValidationIssue(
+                    column=f"{self.column_a}, {self.column_b}",
+                    issue_type="correlation_too_low",
+                    count=1,
+                    severity=Severity.MEDIUM,
+                    details=f"Correlation {correlation:.3f} is below minimum {self.min_correlation}",
+                    expected=f"Correlation >= {self.min_correlation}",
+                )
+            )
+
+        if self.max_correlation is not None and correlation > self.max_correlation:
+            issues.append(
+                ValidationIssue(
+                    column=f"{self.column_a}, {self.column_b}",
+                    issue_type="correlation_too_high",
+                    count=1,
+                    severity=Severity.MEDIUM,
+                    details=f"Correlation {correlation:.3f} exceeds maximum {self.max_correlation}",
+                    expected=f"Correlation <= {self.max_correlation}",
+                )
+            )
+
+        return issues
+
+
+@register_validator
+class ColumnCovarianceValidator(Validator):
+    """Validates covariance between two numeric columns.
+
+    Example:
+        # Check covariance is positive
+        validator = ColumnCovarianceValidator(
+            column_a="x",
+            column_b="y",
+            min_value=0,
+        )
+    """
+
+    name = "column_covariance"
+    category = "multi_column"
+
+    def __init__(
+        self,
+        column_a: str,
+        column_b: str,
+        min_value: float | None = None,
+        max_value: float | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column_a = column_a
+        self.column_b = column_b
+        self.min_value = min_value
+        self.max_value = max_value
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        df = lf.select([self.column_a, self.column_b]).drop_nulls().collect()
+
+        if len(df) < 2:
+            return issues
+
+        # Compute covariance
+        covariance = df.select(
+            pl.cov(self.column_a, self.column_b).alias("cov")
+        )["cov"][0]
+
+        if covariance is None:
+            return issues
+
+        if self.min_value is not None and covariance < self.min_value:
+            issues.append(
+                ValidationIssue(
+                    column=f"{self.column_a}, {self.column_b}",
+                    issue_type="covariance_too_low",
+                    count=1,
+                    severity=Severity.MEDIUM,
+                    details=f"Covariance {covariance:.3f} < minimum {self.min_value}",
+                    expected=f"Covariance >= {self.min_value}",
+                )
+            )
+
+        if self.max_value is not None and covariance > self.max_value:
+            issues.append(
+                ValidationIssue(
+                    column=f"{self.column_a}, {self.column_b}",
+                    issue_type="covariance_too_high",
+                    count=1,
+                    severity=Severity.MEDIUM,
+                    details=f"Covariance {covariance:.3f} > maximum {self.max_value}",
+                    expected=f"Covariance <= {self.max_value}",
+                )
+            )
+
+        return issues
+
+
+@register_validator
+class MultiColumnVarianceValidator(Validator):
+    """Validates variance relationship across multiple columns.
+
+    Example:
+        # Variance of columns should be similar
+        validator = MultiColumnVarianceValidator(
+            columns=["score_1", "score_2", "score_3"],
+            max_variance_ratio=2.0,  # Max ratio between highest and lowest variance
+        )
+    """
+
+    name = "multi_column_variance"
+    category = "multi_column"
+
+    def __init__(
+        self,
+        columns: list[str],
+        max_variance_ratio: float | None = None,
+        min_variance: float | None = None,
+        max_variance: float | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.columns = columns
+        self.max_variance_ratio = max_variance_ratio
+        self.min_variance = min_variance
+        self.max_variance = max_variance
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Calculate variance for each column
+        var_exprs = [pl.col(c).var().alias(f"var_{c}") for c in self.columns]
+        result = lf.select(var_exprs).collect()
+
+        variances = {c: result[f"var_{c}"][0] for c in self.columns}
+        valid_variances = {k: v for k, v in variances.items() if v is not None}
+
+        if len(valid_variances) < 2:
+            return issues
+
+        # Check variance bounds
+        for col, var in valid_variances.items():
+            if self.min_variance is not None and var < self.min_variance:
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="variance_too_low",
+                        count=1,
+                        severity=Severity.MEDIUM,
+                        details=f"Variance {var:.3f} < minimum {self.min_variance}",
+                        expected=f"Variance >= {self.min_variance}",
+                    )
+                )
+
+            if self.max_variance is not None and var > self.max_variance:
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="variance_too_high",
+                        count=1,
+                        severity=Severity.MEDIUM,
+                        details=f"Variance {var:.3f} > maximum {self.max_variance}",
+                        expected=f"Variance <= {self.max_variance}",
+                    )
+                )
+
+        # Check variance ratio
+        if self.max_variance_ratio is not None:
+            max_var = max(valid_variances.values())
+            min_var = min(valid_variances.values())
+
+            if min_var > 0:
+                ratio = max_var / min_var
+                if ratio > self.max_variance_ratio:
+                    issues.append(
+                        ValidationIssue(
+                            column=", ".join(self.columns),
+                            issue_type="variance_ratio_exceeded",
+                            count=1,
+                            severity=Severity.MEDIUM,
+                            details=f"Variance ratio {ratio:.2f} exceeds maximum {self.max_variance_ratio}",
+                            expected=f"Variance ratio <= {self.max_variance_ratio}",
+                        )
+                    )
+
+        return issues