truthound 1.0.8__py3-none-any.whl

truthound/validators/distribution/outlier.py

@@ -0,0 +1,178 @@
+"""Outlier detection validators."""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import (
+    ValidationIssue,
+    Validator,
+    NumericValidatorMixin,
+)
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class OutlierValidator(Validator, NumericValidatorMixin):
+    """Detects statistical outliers using the IQR method."""
+
+    name = "outlier"
+    category = "distribution"
+
+    def __init__(
+        self,
+        iqr_multiplier: float = 1.5,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.iqr_multiplier = iqr_multiplier
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_numeric_columns(lf)
+
+        if not columns:
+            return issues
+
+        # Compute Q1, Q3 for all columns
+        stats_exprs = [pl.len().alias("_total")]
+        for col in columns:
+            stats_exprs.extend([
+                pl.col(col).quantile(0.25).alias(f"_q1_{col}"),
+                pl.col(col).quantile(0.75).alias(f"_q3_{col}"),
+                pl.col(col).count().alias(f"_cnt_{col}"),
+            ])
+
+        stats = lf.select(stats_exprs).collect()
+        total_rows = stats["_total"][0]
+
+        if total_rows < 4:
+            return issues
+
+        # Calculate outliers
+        outlier_exprs = []
+        bounds_info = {}
+
+        for col in columns:
+            q1 = stats[f"_q1_{col}"][0]
+            q3 = stats[f"_q3_{col}"][0]
+            cnt = stats[f"_cnt_{col}"][0]
+
+            if q1 is None or q3 is None or cnt < 4:
+                continue
+
+            iqr = q3 - q1
+            if iqr == 0:
+                continue
+
+            lower = q1 - self.iqr_multiplier * iqr
+            upper = q3 + self.iqr_multiplier * iqr
+
+            bounds_info[col] = (lower, upper, cnt)
+            outlier_exprs.append(
+                ((pl.col(col) < lower) | (pl.col(col) > upper))
+                .sum()
+                .alias(f"_out_{col}")
+            )
+
+        if not outlier_exprs:
+            return issues
+
+        outlier_counts = lf.select(outlier_exprs).collect()
+
+        for col, (lower, upper, cnt) in bounds_info.items():
+            outlier_count = outlier_counts[f"_out_{col}"][0]
+
+            if outlier_count > 0:
+                outlier_pct = outlier_count / cnt
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="outlier",
+                        count=outlier_count,
+                        severity=Severity.MEDIUM if outlier_pct > 0.1 else Severity.LOW,
+                        details=f"IQR bounds: [{lower:.2f}, {upper:.2f}]",
+                        expected=f"[{lower:.2f}, {upper:.2f}]",
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class ZScoreOutlierValidator(Validator, NumericValidatorMixin):
+    """Detects outliers using Z-score method."""
+
+    name = "zscore_outlier"
+    category = "distribution"
+
+    def __init__(
+        self,
+        threshold: float = 3.0,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.threshold = threshold
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_numeric_columns(lf)
+
+        if not columns:
+            return issues
+
+        # Compute mean and std
+        stats_exprs = [pl.len().alias("_total")]
+        for col in columns:
+            stats_exprs.extend([
+                pl.col(col).mean().alias(f"_mean_{col}"),
+                pl.col(col).std().alias(f"_std_{col}"),
+                pl.col(col).count().alias(f"_cnt_{col}"),
+            ])
+
+        stats = lf.select(stats_exprs).collect()
+        total_rows = stats["_total"][0]
+
+        if total_rows < 3:
+            return issues
+
+        # Calculate Z-score outliers
+        outlier_exprs = []
+        stats_info = {}
+
+        for col in columns:
+            mean = stats[f"_mean_{col}"][0]
+            std = stats[f"_std_{col}"][0]
+            cnt = stats[f"_cnt_{col}"][0]
+
+            if mean is None or std is None or std == 0 or cnt < 3:
+                continue
+
+            stats_info[col] = (mean, std, cnt)
+            z_score = (pl.col(col) - mean).abs() / std
+            outlier_exprs.append(
+                (z_score > self.threshold).sum().alias(f"_out_{col}")
+            )
+
+        if not outlier_exprs:
+            return issues
+
+        outlier_counts = lf.select(outlier_exprs).collect()
+
+        for col, (mean, std, cnt) in stats_info.items():
+            outlier_count = outlier_counts[f"_out_{col}"][0]
+
+            if outlier_count > 0:
+                outlier_pct = outlier_count / cnt
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="zscore_outlier",
+                        count=outlier_count,
+                        severity=Severity.MEDIUM if outlier_pct > 0.1 else Severity.LOW,
+                        details=f"|Z-score| > {self.threshold} (mean={mean:.2f}, std={std:.2f})",
+                    )
+                )
+
+        return issues

truthound/validators/distribution/quantile.py

@@ -0,0 +1,80 @@
+"""Quantile validators."""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import (
+    ValidationIssue,
+    Validator,
+    NumericValidatorMixin,
+)
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class QuantileValidator(Validator, NumericValidatorMixin):
+    """Validates that quantile values are within expected range."""
+
+    name = "quantile"
+    category = "distribution"
+
+    def __init__(
+        self,
+        quantile: float = 0.5,
+        min_value: float | None = None,
+        max_value: float | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.quantile = quantile
+        self.min_value = min_value
+        self.max_value = max_value
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_numeric_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.col(c).quantile(self.quantile).alias(f"_q_{c}")
+            for c in columns
+        ]
+        result = lf.select(exprs).collect()
+
+        for col in columns:
+            q_value = result[f"_q_{col}"][0]
+
+            if q_value is None:
+                continue
+
+            if self.min_value is not None and q_value < self.min_value:
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="quantile_low",
+                        count=1,
+                        severity=Severity.MEDIUM,
+                        details=f"Q{self.quantile:.0%}={q_value:.2f} < min {self.min_value}",
+                        expected=f">= {self.min_value}",
+                        actual=q_value,
+                    )
+                )
+
+            if self.max_value is not None and q_value > self.max_value:
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="quantile_high",
+                        count=1,
+                        severity=Severity.MEDIUM,
+                        details=f"Q{self.quantile:.0%}={q_value:.2f} > max {self.max_value}",
+                        expected=f"<= {self.max_value}",
+                        actual=q_value,
+                    )
+                )
+
+        return issues

truthound/validators/distribution/range.py

@@ -0,0 +1,254 @@
+"""Range and boundary validators."""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import (
+    ValidationIssue,
+    Validator,
+    NumericValidatorMixin,
+)
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class BetweenValidator(Validator, NumericValidatorMixin):
+    """Validates that numeric values are within a specified range."""
+
+    name = "between"
+    category = "distribution"
+
+    def __init__(
+        self,
+        min_value: float | None = None,
+        max_value: float | None = None,
+        inclusive: bool = True,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.min_value = min_value
+        self.max_value = max_value
+        self.inclusive = inclusive
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_numeric_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs: list[pl.Expr] = [pl.len().alias("_total")]
+
+        for col in columns:
+            if self.inclusive:
+                below = (pl.col(col) < self.min_value) if self.min_value is not None else pl.lit(False)
+                above = (pl.col(col) > self.max_value) if self.max_value is not None else pl.lit(False)
+            else:
+                below = (pl.col(col) <= self.min_value) if self.min_value is not None else pl.lit(False)
+                above = (pl.col(col) >= self.max_value) if self.max_value is not None else pl.lit(False)
+
+            exprs.append(
+                ((below | above) & pl.col(col).is_not_null()).sum().alias(f"_out_{col}")
+            )
+
+        result = lf.select(exprs).collect()
+        total_rows = result["_total"][0]
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            out_count = result[f"_out_{col}"][0]
+            if out_count > 0:
+                ratio = out_count / total_rows
+                range_str = f"[{self.min_value}, {self.max_value}]"
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="out_of_range",
+                        count=out_count,
+                        severity=self._calculate_severity(ratio, (0.1, 0.05, 0.01)),
+                        details=f"{out_count} values outside {range_str}",
+                        expected=range_str,
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class RangeValidator(Validator, NumericValidatorMixin):
+    """Auto-detects expected ranges based on column names."""
+
+    name = "range"
+    category = "distribution"
+
+    KNOWN_RANGES: dict[str, tuple[float | None, float | None]] = {
+        "age": (0, 150),
+        "price": (0, None),
+        "quantity": (0, None),
+        "count": (0, None),
+        "amount": (0, None),
+        "percentage": (0, 100),
+        "percent": (0, 100),
+        "pct": (0, 100),
+        "rate": (0, 100),
+        "score": (0, 100),
+        "rating": (0, 5),
+        "year": (1900, 2100),
+        "month": (1, 12),
+        "day": (1, 31),
+        "hour": (0, 23),
+        "minute": (0, 59),
+        "second": (0, 59),
+    }
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_numeric_columns(lf)
+
+        if not columns:
+            return issues
+
+        # Collect data once
+        df = lf.collect()
+        total_rows = len(df)
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            col_lower = col.lower()
+
+            # Find matching range
+            min_val, max_val = None, None
+            for pattern, (pmin, pmax) in self.KNOWN_RANGES.items():
+                if pattern in col_lower:
+                    min_val, max_val = pmin, pmax
+                    break
+
+            if min_val is None and max_val is None:
+                continue
+
+            col_data = df.get_column(col)
+            out_count = 0
+
+            if min_val is not None:
+                out_count += col_data.filter(col_data < min_val).drop_nulls().len()
+            if max_val is not None:
+                out_count += col_data.filter(col_data > max_val).drop_nulls().len()
+
+            if out_count > 0:
+                oor_pct = out_count / total_rows
+                range_desc = ""
+                if min_val is not None and max_val is not None:
+                    range_desc = f"[{min_val}, {max_val}]"
+                elif min_val is not None:
+                    range_desc = f">= {min_val}"
+                else:
+                    range_desc = f"<= {max_val}"
+
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="out_of_range",
+                        count=out_count,
+                        severity=self._calculate_severity(oor_pct, (0.1, 0.05, 0.01)),
+                        details=f"Expected {range_desc}",
+                        expected=range_desc,
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class PositiveValidator(Validator, NumericValidatorMixin):
+    """Validates that numeric values are positive (> 0)."""
+
+    name = "positive"
+    category = "distribution"
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_numeric_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.len().alias("_total"),
+            *[
+                ((pl.col(c) <= 0) & pl.col(c).is_not_null()).sum().alias(f"_neg_{c}")
+                for c in columns
+            ],
+        ]
+        result = lf.select(exprs).collect()
+        total_rows = result["_total"][0]
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            neg_count = result[f"_neg_{col}"][0]
+            if neg_count > 0:
+                ratio = neg_count / total_rows
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="not_positive",
+                        count=neg_count,
+                        severity=self._calculate_severity(ratio),
+                        details=f"{neg_count} non-positive values",
+                        expected="> 0",
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class NonNegativeValidator(Validator, NumericValidatorMixin):
+    """Validates that numeric values are non-negative (>= 0)."""
+
+    name = "non_negative"
+    category = "distribution"
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_numeric_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.len().alias("_total"),
+            *[
+                ((pl.col(c) < 0) & pl.col(c).is_not_null()).sum().alias(f"_neg_{c}")
+                for c in columns
+            ],
+        ]
+        result = lf.select(exprs).collect()
+        total_rows = result["_total"][0]
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            neg_count = result[f"_neg_{col}"][0]
+            if neg_count > 0:
+                ratio = neg_count / total_rows
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="negative",
+                        count=neg_count,
+                        severity=self._calculate_severity(ratio),
+                        details=f"{neg_count} negative values",
+                        expected=">= 0",
+                    )
+                )
+
+        return issues

truthound/validators/distribution/set.py

@@ -0,0 +1,125 @@
+"""Set membership validators."""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class InSetValidator(Validator):
+    """Validates that values are in a specified set."""
+
+    name = "in_set"
+    category = "distribution"
+
+    def __init__(
+        self,
+        allowed_values: list[Any],
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.allowed_values = allowed_values
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_target_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.len().alias("_total"),
+            *[
+                (
+                    ~pl.col(c).is_in(self.allowed_values) & pl.col(c).is_not_null()
+                ).sum().alias(f"_not_in_{c}")
+                for c in columns
+            ],
+        ]
+        result = lf.select(exprs).collect()
+        total_rows = result["_total"][0]
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            not_in_count = result[f"_not_in_{col}"][0]
+            if not_in_count > 0:
+                ratio = not_in_count / total_rows
+                allowed_str = str(self.allowed_values[:5])
+                if len(self.allowed_values) > 5:
+                    allowed_str = allowed_str[:-1] + ", ...]"
+
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="not_in_set",
+                        count=not_in_count,
+                        severity=self._calculate_severity(ratio),
+                        details=f"{not_in_count} values not in allowed set",
+                        expected=allowed_str,
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class NotInSetValidator(Validator):
+    """Validates that values are NOT in a specified set (forbidden values)."""
+
+    name = "not_in_set"
+    category = "distribution"
+
+    def __init__(
+        self,
+        forbidden_values: list[Any],
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.forbidden_values = forbidden_values
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_target_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.len().alias("_total"),
+            *[
+                pl.col(c).is_in(self.forbidden_values).sum().alias(f"_in_{c}")
+                for c in columns
+            ],
+        ]
+        result = lf.select(exprs).collect()
+        total_rows = result["_total"][0]
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            in_count = result[f"_in_{col}"][0]
+            if in_count > 0:
+                ratio = in_count / total_rows
+                forbidden_str = str(self.forbidden_values[:5])
+                if len(self.forbidden_values) > 5:
+                    forbidden_str = forbidden_str[:-1] + ", ...]"
+
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="in_forbidden_set",
+                        count=in_count,
+                        severity=Severity.HIGH,
+                        details=f"{in_count} forbidden values found",
+                        expected=f"Not in {forbidden_str}",
+                    )
+                )
+
+        return issues