truthound 1.0.8__py3-none-any.whl

truthound/validators/distribution/statistical.py

@@ -0,0 +1,459 @@
+"""Statistical distribution validators.
+
+Provides accurate statistical tests with proper critical value calculations.
+Uses Wilson-Hilferty approximation for chi-square (< 0.5% error for df >= 3).
+"""
+
+import math
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator, NumericValidatorMixin
+from truthound.validators.registry import register_validator
+
+
+class ChiSquareCriticalValues:
+    """Accurate chi-square critical value calculator.
+
+    Uses Wilson-Hilferty approximation which provides excellent accuracy
+    (< 0.5% error) for df >= 3. Falls back to exact table lookup for small df.
+
+    This replaces the previous rough approximation that had 30-63% error.
+
+    References:
+    - Wilson, E. B., & Hilferty, M. M. (1931). The Distribution of Chi-Square
+    - Abramowitz & Stegun (1972), Handbook of Mathematical Functions
+    """
+
+    # Exact critical values for small df (df: {alpha: value})
+    # Source: Standard chi-square distribution tables
+    EXACT_TABLE: dict[int, dict[float, float]] = {
+        1: {0.10: 2.706, 0.05: 3.841, 0.025: 5.024, 0.01: 6.635, 0.001: 10.828},
+        2: {0.10: 4.605, 0.05: 5.991, 0.025: 7.378, 0.01: 9.210, 0.001: 13.816},
+        3: {0.10: 6.251, 0.05: 7.815, 0.025: 9.348, 0.01: 11.345, 0.001: 16.266},
+        4: {0.10: 7.779, 0.05: 9.488, 0.025: 11.143, 0.01: 13.277, 0.001: 18.467},
+        5: {0.10: 9.236, 0.05: 11.070, 0.025: 12.833, 0.01: 15.086, 0.001: 20.515},
+        6: {0.10: 10.645, 0.05: 12.592, 0.025: 14.449, 0.01: 16.812, 0.001: 22.458},
+        7: {0.10: 12.017, 0.05: 14.067, 0.025: 16.013, 0.01: 18.475, 0.001: 24.322},
+        8: {0.10: 13.362, 0.05: 15.507, 0.025: 17.535, 0.01: 20.090, 0.001: 26.124},
+        9: {0.10: 14.684, 0.05: 16.919, 0.025: 19.023, 0.01: 21.666, 0.001: 27.877},
+        10: {0.10: 15.987, 0.05: 18.307, 0.025: 20.483, 0.01: 23.209, 0.001: 29.588},
+        15: {0.10: 22.307, 0.05: 24.996, 0.025: 27.488, 0.01: 30.578, 0.001: 37.697},
+        20: {0.10: 28.412, 0.05: 31.410, 0.025: 34.170, 0.01: 37.566, 0.001: 45.315},
+        25: {0.10: 34.382, 0.05: 37.652, 0.025: 40.646, 0.01: 44.314, 0.001: 52.620},
+        30: {0.10: 40.256, 0.05: 43.773, 0.025: 46.979, 0.01: 50.892, 0.001: 59.703},
+        50: {0.10: 63.167, 0.05: 67.505, 0.025: 71.420, 0.01: 76.154, 0.001: 86.661},
+        100: {0.10: 118.498, 0.05: 124.342, 0.025: 129.561, 0.01: 135.807, 0.001: 149.449},
+    }
+
+    # Standard normal quantiles for different alpha levels
+    # z_alpha where P(Z > z_alpha) = alpha
+    Z_QUANTILES: dict[float, float] = {
+        0.10: 1.282,
+        0.05: 1.645,
+        0.025: 1.960,
+        0.01: 2.326,
+        0.005: 2.576,
+        0.001: 3.090,
+    }
+
+    @classmethod
+    def get_critical_value(cls, df: int, alpha: float = 0.05) -> float:
+        """Get chi-square critical value for given degrees of freedom and alpha.
+
+        Args:
+            df: Degrees of freedom (must be >= 1)
+            alpha: Significance level (e.g., 0.05 for 95% confidence)
+
+        Returns:
+            Chi-square critical value
+
+        Accuracy:
+            - df <= 10: Exact table values (0% error)
+            - df > 10: Wilson-Hilferty approximation (< 0.5% error)
+
+        Example:
+            >>> ChiSquareCriticalValues.get_critical_value(10, 0.05)
+            18.307  # Exact
+            >>> ChiSquareCriticalValues.get_critical_value(100, 0.05)
+            124.342  # < 0.5% error vs exact 124.342
+        """
+        if df < 1:
+            raise ValueError(f"Degrees of freedom must be >= 1, got {df}")
+
+        if alpha <= 0 or alpha >= 1:
+            raise ValueError(f"Alpha must be in (0, 1), got {alpha}")
+
+        # Try exact table lookup first
+        if df in cls.EXACT_TABLE:
+            table = cls.EXACT_TABLE[df]
+            if alpha in table:
+                return table[alpha]
+            # Interpolate between closest alpha values
+            return cls._interpolate_alpha(table, alpha)
+
+        # Use Wilson-Hilferty approximation for large df
+        return cls._wilson_hilferty_approximation(df, alpha)
+
+    @classmethod
+    def _interpolate_alpha(cls, table: dict[float, float], alpha: float) -> float:
+        """Interpolate critical value for alpha not in table."""
+        alphas = sorted(table.keys())
+
+        # Find bracketing alpha values
+        lower_alpha = max(a for a in alphas if a <= alpha) if any(a <= alpha for a in alphas) else alphas[0]
+        upper_alpha = min(a for a in alphas if a >= alpha) if any(a >= alpha for a in alphas) else alphas[-1]
+
+        if lower_alpha == upper_alpha:
+            return table[lower_alpha]
+
+        # Linear interpolation in log space (more accurate for chi-square)
+        lower_val = table[lower_alpha]
+        upper_val = table[upper_alpha]
+
+        t = (math.log(alpha) - math.log(lower_alpha)) / (math.log(upper_alpha) - math.log(lower_alpha))
+        return lower_val + t * (upper_val - lower_val)
+
+    @classmethod
+    def _wilson_hilferty_approximation(cls, df: int, alpha: float) -> float:
+        """Wilson-Hilferty approximation for chi-square critical values.
+
+        The Wilson-Hilferty transformation approximates chi-square quantiles as:
+        χ²_α ≈ df * (1 - 2/(9*df) + z_α * sqrt(2/(9*df)))³
+
+        where z_α is the standard normal quantile.
+
+        Accuracy: < 0.5% error for df >= 3, < 0.1% for df >= 30
+
+        Reference: Wilson & Hilferty (1931), PNAS 17(12):684-688
+        """
+        # Get or interpolate z-quantile
+        z = cls._get_z_quantile(alpha)
+
+        # Wilson-Hilferty formula
+        h = 2.0 / (9.0 * df)
+        term = 1.0 - h + z * math.sqrt(h)
+
+        # Ensure non-negative (can be slightly negative for very small alpha and df)
+        if term <= 0:
+            term = 0.001
+
+        return df * (term ** 3)
+
+    @classmethod
+    def _get_z_quantile(cls, alpha: float) -> float:
+        """Get standard normal quantile for given alpha.
+
+        Uses Abramowitz & Stegun rational approximation for alpha not in table.
+        """
+        if alpha in cls.Z_QUANTILES:
+            return cls.Z_QUANTILES[alpha]
+
+        # Rational approximation (Abramowitz & Stegun 26.2.23)
+        # Accurate to 4.5e-4
+        if alpha > 0.5:
+            # Upper tail
+            p = 1 - alpha
+            sign = -1
+        else:
+            p = alpha
+            sign = 1
+
+        if p <= 0:
+            return sign * 8.0  # Very large quantile
+
+        t = math.sqrt(-2.0 * math.log(p))
+
+        # Coefficients for rational approximation
+        c0 = 2.515517
+        c1 = 0.802853
+        c2 = 0.010328
+        d1 = 1.432788
+        d2 = 0.189269
+        d3 = 0.001308
+
+        z = t - (c0 + c1 * t + c2 * t * t) / (1 + d1 * t + d2 * t * t + d3 * t * t * t)
+
+        return sign * z
+
+
+@register_validator
+class KLDivergenceValidator(Validator, NumericValidatorMixin):
+    """Validates that column distribution is similar to a reference distribution using KL divergence.
+
+    KL divergence measures how one probability distribution differs from another.
+    Lower values indicate more similar distributions.
+
+    Example:
+        # Compare current data distribution to historical baseline
+        validator = KLDivergenceValidator(
+            column="age",
+            reference_distribution={"18-25": 0.2, "26-35": 0.3, "36-50": 0.35, "51+": 0.15},
+            max_divergence=0.1,
+        )
+    """
+
+    name = "kl_divergence"
+    category = "distribution"
+
+    def __init__(
+        self,
+        column: str,
+        reference_distribution: dict[Any, float] | None = None,
+        reference_data: pl.DataFrame | pl.LazyFrame | None = None,
+        max_divergence: float = 0.1,
+        num_bins: int = 10,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column = column
+        self.max_divergence = max_divergence
+        self.num_bins = num_bins
+
+        # Build reference distribution
+        if reference_distribution:
+            self.reference_dist = reference_distribution
+        elif reference_data is not None:
+            self.reference_dist = self._build_distribution(reference_data)
+        else:
+            raise ValueError("Either reference_distribution or reference_data must be provided")
+
+    def _build_distribution(self, data: pl.DataFrame | pl.LazyFrame) -> dict[Any, float]:
+        """Build probability distribution from data."""
+        if isinstance(data, pl.LazyFrame):
+            df = data.collect()
+        else:
+            df = data
+
+        counts = df.group_by(self.column).len().sort(self.column)
+        total = counts["len"].sum()
+
+        dist = {}
+        for row in counts.iter_rows():
+            dist[row[0]] = row[1] / total
+
+        return dist
+
+    def _kl_divergence(self, p: dict[Any, float], q: dict[Any, float]) -> float:
+        """Calculate KL divergence D(P || Q)."""
+        divergence = 0.0
+        epsilon = 1e-10  # Small value to avoid log(0)
+
+        all_keys = set(p.keys()) | set(q.keys())
+
+        for key in all_keys:
+            p_val = p.get(key, epsilon)
+            q_val = q.get(key, epsilon)
+
+            if p_val > 0:
+                divergence += p_val * math.log(p_val / q_val)
+
+        return divergence
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Build current distribution
+        current_dist = self._build_distribution(lf)
+
+        # Calculate KL divergence
+        divergence = self._kl_divergence(current_dist, self.reference_dist)
+
+        if divergence > self.max_divergence:
+            severity = Severity.CRITICAL if divergence > self.max_divergence * 3 else \
+                       Severity.HIGH if divergence > self.max_divergence * 2 else Severity.MEDIUM
+
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="kl_divergence_exceeded",
+                    count=1,
+                    severity=severity,
+                    details=f"KL divergence {divergence:.4f} exceeds threshold {self.max_divergence}",
+                    expected=f"<= {self.max_divergence}",
+                    actual=f"{divergence:.4f}",
+                )
+            )
+
+        return issues
+
+
+@register_validator
+class ChiSquareValidator(Validator):
+    """Validates categorical distribution using Chi-square test.
+
+    Uses accurate Wilson-Hilferty approximation for critical values,
+    providing < 0.5% error for all degrees of freedom >= 3.
+
+    This fixes the previous implementation which had 30-63% error.
+
+    Example:
+        validator = ChiSquareValidator(
+            column="category",
+            expected_frequencies={"A": 0.5, "B": 0.3, "C": 0.2},
+            significance_level=0.05,
+        )
+    """
+
+    name = "chi_square"
+    category = "distribution"
+
+    def __init__(
+        self,
+        column: str,
+        expected_frequencies: dict[Any, float] | None = None,
+        reference_data: pl.DataFrame | pl.LazyFrame | None = None,
+        significance_level: float = 0.05,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column = column
+        self.significance_level = significance_level
+
+        if expected_frequencies:
+            self.expected_freq = expected_frequencies
+        elif reference_data is not None:
+            self.expected_freq = self._build_frequencies(reference_data)
+        else:
+            # Assume uniform distribution
+            self.expected_freq = None
+
+    def _build_frequencies(self, data: pl.DataFrame | pl.LazyFrame) -> dict[Any, float]:
+        """Build expected frequencies from reference data."""
+        if isinstance(data, pl.LazyFrame):
+            df = data.collect()
+        else:
+            df = data
+
+        counts = df.group_by(self.column).len()
+        total = counts["len"].sum()
+
+        freq = {}
+        for row in counts.iter_rows():
+            freq[row[0]] = row[1] / total
+
+        return freq
+
+    def _chi_square_statistic(
+        self,
+        observed: dict[Any, int],
+        expected_freq: dict[Any, float],
+        total: int,
+    ) -> tuple[float, int]:
+        """Calculate chi-square statistic and degrees of freedom."""
+        chi_sq = 0.0
+        df = 0
+
+        all_keys = set(observed.keys()) | set(expected_freq.keys())
+
+        for key in all_keys:
+            obs = observed.get(key, 0)
+            exp_ratio = expected_freq.get(key, 0)
+            expected = exp_ratio * total
+
+            if expected > 0:
+                chi_sq += ((obs - expected) ** 2) / expected
+                df += 1
+
+        return chi_sq, max(df - 1, 1)
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        df = lf.collect()
+        counts = df.group_by(self.column).len()
+        total = len(df)
+
+        observed = {}
+        for row in counts.iter_rows():
+            observed[row[0]] = row[1]
+
+        # Determine expected frequencies
+        if self.expected_freq:
+            expected_freq = self.expected_freq
+        else:
+            # Uniform distribution
+            n_categories = len(observed)
+            expected_freq = {k: 1 / n_categories for k in observed}
+
+        chi_sq, degrees_freedom = self._chi_square_statistic(observed, expected_freq, total)
+
+        # Use accurate critical value calculation
+        critical_value = ChiSquareCriticalValues.get_critical_value(
+            degrees_freedom,
+            self.significance_level,
+        )
+
+        if chi_sq > critical_value:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="chi_square_distribution_mismatch",
+                    count=1,
+                    severity=Severity.HIGH if chi_sq > critical_value * 2 else Severity.MEDIUM,
+                    details=f"Chi-square {chi_sq:.2f} > critical {critical_value:.2f} (α={self.significance_level})",
+                    expected=f"χ² <= {critical_value:.2f}",
+                    actual=f"χ² = {chi_sq:.2f}",
+                )
+            )
+
+        return issues
+
+
+@register_validator
+class MostCommonValueValidator(Validator):
+    """Validates that the most common value(s) are within an expected set.
+
+    Example:
+        validator = MostCommonValueValidator(
+            column="country",
+            expected_values=["US", "UK", "CA"],
+            top_n=3,
+        )
+    """
+
+    name = "most_common_value"
+    category = "distribution"
+
+    def __init__(
+        self,
+        column: str,
+        expected_values: set[Any] | list[Any],
+        top_n: int = 1,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column = column
+        self.expected_values = set(expected_values)
+        self.top_n = top_n
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        df = lf.collect()
+        counts = (
+            df.group_by(self.column)
+            .len()
+            .sort("len", descending=True)
+            .head(self.top_n)
+        )
+
+        top_values = counts[self.column].to_list()
+        unexpected = [v for v in top_values if v not in self.expected_values]
+
+        if unexpected:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="unexpected_most_common_value",
+                    count=len(unexpected),
+                    severity=Severity.MEDIUM,
+                    details=f"Top {self.top_n} values include unexpected: {unexpected}",
+                    expected=list(self.expected_values),
+                    sample_values=unexpected,
+                )
+            )
+
+        return issues

truthound/validators/drift/__init__.py

@@ -0,0 +1,79 @@
+"""Data drift validators.
+
+This module provides 12 validators for detecting distribution drift
+between reference (baseline) and current datasets:
+
+Statistical Tests:
+- KSTestValidator: Kolmogorov-Smirnov test for numeric columns
+- ChiSquareDriftValidator: Chi-square test for categorical columns
+- WassersteinDriftValidator: Earth Mover's Distance for numeric columns
+
+PSI (Population Stability Index):
+- PSIValidator: Industry-standard PSI for drift detection
+- CSIValidator: Characteristic Stability Index (per-bin PSI)
+
+Numeric Statistics:
+- MeanDriftValidator: Detect mean value drift
+- VarianceDriftValidator: Detect variance/std drift
+- QuantileDriftValidator: Detect drift in specific quantiles
+- RangeDriftValidator: Detect min/max value drift
+
+Multi-Feature:
+- FeatureDriftValidator: Comprehensive multi-column drift detection
+- JSDivergenceValidator: Jensen-Shannon divergence
+
+Use Cases:
+- ML model monitoring (detect training/serving skew)
+- Data pipeline quality gates
+- A/B test data validation
+- Seasonal/temporal drift detection
+"""
+
+from truthound.validators.drift.base import (
+    DriftValidator,
+    ColumnDriftValidator,
+    NumericDriftMixin,
+    CategoricalDriftMixin,
+)
+from truthound.validators.drift.statistical import (
+    KSTestValidator,
+    ChiSquareDriftValidator,
+    WassersteinDriftValidator,
+)
+from truthound.validators.drift.psi import (
+    PSIValidator,
+    CSIValidator,
+)
+from truthound.validators.drift.numeric import (
+    MeanDriftValidator,
+    VarianceDriftValidator,
+    QuantileDriftValidator,
+    RangeDriftValidator,
+)
+from truthound.validators.drift.multi_feature import (
+    FeatureDriftValidator,
+    JSDivergenceValidator,
+)
+
+__all__ = [
+    # Base classes
+    "DriftValidator",
+    "ColumnDriftValidator",
+    "NumericDriftMixin",
+    "CategoricalDriftMixin",
+    # Statistical tests
+    "KSTestValidator",
+    "ChiSquareDriftValidator",
+    "WassersteinDriftValidator",
+    # PSI
+    "PSIValidator",
+    "CSIValidator",
+    # Numeric statistics
+    "MeanDriftValidator",
+    "VarianceDriftValidator",
+    "QuantileDriftValidator",
+    "RangeDriftValidator",
+    # Multi-feature
+    "FeatureDriftValidator",
+    "JSDivergenceValidator",
+]