truthound 1.0.8__py3-none-any.whl

truthound/validators/uniqueness/distinct_values.py

@@ -0,0 +1,253 @@
+"""Distinct value validators."""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class DistinctValuesInSetValidator(Validator):
+    """Validates that all distinct values are within an allowed set.
+
+    Example:
+        # All status values should be in the allowed list
+        validator = DistinctValuesInSetValidator(
+            column="status",
+            allowed_values=["pending", "active", "completed", "cancelled"],
+        )
+    """
+
+    name = "distinct_values_in_set"
+    category = "uniqueness"
+
+    def __init__(
+        self,
+        column: str,
+        allowed_values: set[Any] | list[Any],
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column = column
+        self.allowed_values = set(allowed_values)
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Get distinct values
+        distinct_values = set(
+            lf.select(pl.col(self.column).drop_nulls().unique())
+            .collect()
+            .to_series()
+            .to_list()
+        )
+
+        # Find values not in allowed set
+        unexpected = distinct_values - self.allowed_values
+
+        if unexpected:
+            samples = list(unexpected)[: self.config.sample_size]
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="distinct_values_not_in_set",
+                    count=len(unexpected),
+                    severity=Severity.HIGH,
+                    details=f"{len(unexpected)} unexpected distinct values found",
+                    expected=f"Values in {list(self.allowed_values)[:5]}{'...' if len(self.allowed_values) > 5 else ''}",
+                    sample_values=samples,
+                )
+            )
+
+        return issues
+
+
+@register_validator
+class DistinctValuesEqualSetValidator(Validator):
+    """Validates that distinct values exactly match an expected set.
+
+    Example:
+        # Days of week should be exactly these 7 values
+        validator = DistinctValuesEqualSetValidator(
+            column="day_of_week",
+            expected_values=["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"],
+        )
+    """
+
+    name = "distinct_values_equal_set"
+    category = "uniqueness"
+
+    def __init__(
+        self,
+        column: str,
+        expected_values: set[Any] | list[Any],
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column = column
+        self.expected_values = set(expected_values)
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Get distinct values
+        distinct_values = set(
+            lf.select(pl.col(self.column).drop_nulls().unique())
+            .collect()
+            .to_series()
+            .to_list()
+        )
+
+        # Check for missing values
+        missing = self.expected_values - distinct_values
+        if missing:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="distinct_values_missing",
+                    count=len(missing),
+                    severity=Severity.HIGH,
+                    details=f"Missing {len(missing)} expected distinct values",
+                    expected=list(self.expected_values),
+                    sample_values=list(missing)[: self.config.sample_size],
+                )
+            )
+
+        # Check for unexpected values
+        unexpected = distinct_values - self.expected_values
+        if unexpected:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="distinct_values_unexpected",
+                    count=len(unexpected),
+                    severity=Severity.HIGH,
+                    details=f"Found {len(unexpected)} unexpected distinct values",
+                    expected=list(self.expected_values),
+                    sample_values=list(unexpected)[: self.config.sample_size],
+                )
+            )
+
+        return issues
+
+
+@register_validator
+class DistinctValuesContainSetValidator(Validator):
+    """Validates that distinct values contain all values from a required set.
+
+    Example:
+        # Product categories must include these core categories
+        validator = DistinctValuesContainSetValidator(
+            column="category",
+            required_values=["Electronics", "Clothing", "Food"],
+        )
+    """
+
+    name = "distinct_values_contain_set"
+    category = "uniqueness"
+
+    def __init__(
+        self,
+        column: str,
+        required_values: set[Any] | list[Any],
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column = column
+        self.required_values = set(required_values)
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Get distinct values
+        distinct_values = set(
+            lf.select(pl.col(self.column).drop_nulls().unique())
+            .collect()
+            .to_series()
+            .to_list()
+        )
+
+        # Check if all required values are present
+        missing = self.required_values - distinct_values
+
+        if missing:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="distinct_values_missing_required",
+                    count=len(missing),
+                    severity=Severity.HIGH,
+                    details=f"Missing {len(missing)} required values",
+                    expected=list(self.required_values),
+                    sample_values=list(missing)[: self.config.sample_size],
+                )
+            )
+
+        return issues
+
+
+@register_validator
+class DistinctCountBetweenValidator(Validator):
+    """Validates that distinct value count is within a range.
+
+    Example:
+        # Should have between 5 and 20 unique product categories
+        validator = DistinctCountBetweenValidator(
+            column="category",
+            min_count=5,
+            max_count=20,
+        )
+    """
+
+    name = "distinct_count_between"
+    category = "uniqueness"
+
+    def __init__(
+        self,
+        column: str,
+        min_count: int | None = None,
+        max_count: int | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column = column
+        self.min_count = min_count
+        self.max_count = max_count
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        distinct_count = lf.select(pl.col(self.column).n_unique()).collect().item()
+
+        # Check minimum
+        if self.min_count is not None and distinct_count < self.min_count:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="distinct_count_too_low",
+                    count=self.min_count - distinct_count,
+                    severity=Severity.HIGH,
+                    details=f"Distinct count {distinct_count} below minimum {self.min_count}",
+                    expected=f">= {self.min_count}",
+                    actual=distinct_count,
+                )
+            )
+
+        # Check maximum
+        if self.max_count is not None and distinct_count > self.max_count:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="distinct_count_too_high",
+                    count=distinct_count - self.max_count,
+                    severity=Severity.HIGH,
+                    details=f"Distinct count {distinct_count} above maximum {self.max_count}",
+                    expected=f"<= {self.max_count}",
+                    actual=distinct_count,
+                )
+            )
+
+        return issues

truthound/validators/uniqueness/duplicate.py

@@ -0,0 +1,118 @@
+"""Duplicate row validators."""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class DuplicateValidator(Validator):
+    """Detects duplicate rows in the dataset."""
+
+    name = "duplicate"
+    category = "uniqueness"
+
+    def __init__(
+        self,
+        subset: list[str] | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.subset = subset
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        columns = self.subset or self._get_target_columns(lf)
+        if not columns:
+            return issues
+
+        # Count duplicates
+        result = lf.select([
+            pl.len().alias("_total"),
+            pl.struct(columns).is_duplicated().sum().alias("_dup"),
+        ]).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        dup_count = result["_dup"][0]
+        if dup_count > 0:
+            dup_pct = dup_count / total_rows
+            col_desc = ", ".join(columns[:3]) + ("..." if len(columns) > 3 else "")
+            issues.append(
+                ValidationIssue(
+                    column=f"[{col_desc}]",
+                    issue_type="duplicate_row",
+                    count=dup_count,
+                    severity=self._calculate_severity(
+                        dup_pct, thresholds=(0.3, 0.1, 0.01)
+                    ),
+                    details=f"{dup_count} duplicate rows ({dup_pct:.1%})",
+                )
+            )
+
+        return issues
+
+
+@register_validator
+class DuplicateWithinGroupValidator(Validator):
+    """Detects duplicate values within groups.
+
+    Example:
+        # Check for duplicate order_id within same customer
+        validator = DuplicateWithinGroupValidator(
+            group_by=["customer_id"],
+            check_column="order_id",
+        )
+    """
+
+    name = "duplicate_within_group"
+    category = "uniqueness"
+
+    def __init__(
+        self,
+        group_by: list[str],
+        check_column: str,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.group_by = group_by
+        self.check_column = check_column
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Count duplicates within groups
+        dup_count = (
+            lf.group_by(self.group_by)
+            .agg(
+                (pl.col(self.check_column).count() - pl.col(self.check_column).n_unique())
+                .alias("_dup")
+            )
+            .select(pl.col("_dup").sum())
+            .collect()
+            .item()
+        )
+
+        if dup_count > 0:
+            total_rows = lf.select(pl.len()).collect().item()
+            dup_pct = dup_count / total_rows if total_rows > 0 else 0
+            group_desc = ", ".join(self.group_by)
+
+            issues.append(
+                ValidationIssue(
+                    column=self.check_column,
+                    issue_type="duplicate_within_group",
+                    count=dup_count,
+                    severity=Severity.HIGH,
+                    details=f"{dup_count} duplicates within [{group_desc}] groups",
+                )
+            )
+
+        return issues

truthound/validators/uniqueness/primary_key.py

@@ -0,0 +1,140 @@
+"""Primary key validators."""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class PrimaryKeyValidator(Validator):
+    """Validates that a column can serve as a primary key (unique and not null)."""
+
+    name = "primary_key"
+    category = "uniqueness"
+
+    def __init__(
+        self,
+        column: str,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column = column
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        result = lf.select([
+            pl.len().alias("_total"),
+            pl.col(self.column).null_count().alias("_null"),
+            # Count unique non-null values only
+            pl.col(self.column).drop_nulls().n_unique().alias("_unique"),
+            pl.col(self.column).count().alias("_count"),
+        ]).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        null_count = result["_null"][0]
+        unique_count = result["_unique"][0]
+        non_null_count = result["_count"][0]
+
+        # Check for nulls
+        if null_count > 0:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="primary_key_null",
+                    count=null_count,
+                    severity=Severity.CRITICAL,
+                    details=f"Primary key has {null_count} null values",
+                )
+            )
+
+        # Check for duplicates (non_null_count - unique gives duplicate rows)
+        dup_count = non_null_count - unique_count
+        if dup_count > 0:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="primary_key_duplicate",
+                    count=dup_count,
+                    severity=Severity.CRITICAL,
+                    details=f"Primary key has {dup_count} duplicate values",
+                )
+            )
+
+        return issues
+
+
+@register_validator
+class CompoundKeyValidator(Validator):
+    """Validates that a combination of columns forms a unique key."""
+
+    name = "compound_key"
+    category = "uniqueness"
+
+    def __init__(
+        self,
+        columns: list[str],
+        allow_nulls: bool = False,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.key_columns = columns
+        self.allow_nulls = allow_nulls
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Check for nulls in any key column
+        if not self.allow_nulls:
+            null_exprs = [
+                pl.col(c).null_count().alias(f"_null_{c}")
+                for c in self.key_columns
+            ]
+            null_result = lf.select(null_exprs).collect()
+
+            for col in self.key_columns:
+                null_count = null_result[f"_null_{col}"][0]
+                if null_count > 0:
+                    issues.append(
+                        ValidationIssue(
+                            column=col,
+                            issue_type="compound_key_null",
+                            count=null_count,
+                            severity=Severity.CRITICAL,
+                            details=f"Compound key column has {null_count} nulls",
+                        )
+                    )
+
+        # Check for duplicates
+        result = lf.select([
+            pl.len().alias("_total"),
+            pl.struct(self.key_columns).n_unique().alias("_unique"),
+        ]).collect()
+
+        total_rows = result["_total"][0]
+        unique_count = result["_unique"][0]
+
+        if total_rows == 0:
+            return issues
+
+        dup_count = total_rows - unique_count
+        if dup_count > 0:
+            key_desc = ", ".join(self.key_columns)
+            issues.append(
+                ValidationIssue(
+                    column=f"[{key_desc}]",
+                    issue_type="compound_key_duplicate",
+                    count=dup_count,
+                    severity=Severity.CRITICAL,
+                    details=f"Compound key has {dup_count} duplicate combinations",
+                )
+            )
+
+        return issues

truthound/validators/uniqueness/unique.py

@@ -0,0 +1,191 @@
+"""Unique value validators."""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class UniqueValidator(Validator):
+    """Validates that column values are unique (no duplicates)."""
+
+    name = "unique"
+    category = "uniqueness"
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_target_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.len().alias("_total"),
+            *[pl.col(c).n_unique().alias(f"_unique_{c}") for c in columns],
+            *[pl.col(c).count().alias(f"_count_{c}") for c in columns],
+        ]
+        result = lf.select(exprs).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            unique_count = result[f"_unique_{col}"][0]
+            non_null_count = result[f"_count_{col}"][0]
+            duplicate_count = non_null_count - unique_count
+
+            if duplicate_count > 0:
+                dup_pct = duplicate_count / non_null_count if non_null_count > 0 else 0
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="unique_violation",
+                        count=duplicate_count,
+                        severity=self._calculate_severity(
+                            dup_pct, thresholds=(0.3, 0.1, 0.01)
+                        ),
+                        details=f"{duplicate_count} duplicate values ({dup_pct:.1%})",
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class UniqueRatioValidator(Validator):
+    """Validates that uniqueness ratio is within expected range."""
+
+    name = "unique_ratio"
+    category = "uniqueness"
+
+    def __init__(
+        self,
+        min_ratio: float | None = None,
+        max_ratio: float | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.min_ratio = min_ratio
+        self.max_ratio = max_ratio
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_target_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.len().alias("_total"),
+            *[pl.col(c).n_unique().alias(f"_unique_{c}") for c in columns],
+            *[pl.col(c).count().alias(f"_count_{c}") for c in columns],
+        ]
+        result = lf.select(exprs).collect()
+
+        total_rows = result["_total"][0]
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            unique_count = result[f"_unique_{col}"][0]
+            non_null_count = result[f"_count_{col}"][0]
+
+            if non_null_count == 0:
+                continue
+
+            ratio = unique_count / non_null_count
+
+            if self.min_ratio is not None and ratio < self.min_ratio:
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="unique_ratio_low",
+                        count=int((self.min_ratio - ratio) * non_null_count),
+                        severity=Severity.MEDIUM,
+                        details=f"Unique ratio {ratio:.1%} < min {self.min_ratio:.1%}",
+                        expected=f">= {self.min_ratio:.1%}",
+                        actual=f"{ratio:.1%}",
+                    )
+                )
+
+            if self.max_ratio is not None and ratio > self.max_ratio:
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="unique_ratio_high",
+                        count=int((ratio - self.max_ratio) * non_null_count),
+                        severity=Severity.MEDIUM,
+                        details=f"Unique ratio {ratio:.1%} > max {self.max_ratio:.1%}",
+                        expected=f"<= {self.max_ratio:.1%}",
+                        actual=f"{ratio:.1%}",
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class DistinctCountValidator(Validator):
+    """Validates that distinct count is within expected range."""
+
+    name = "distinct_count"
+    category = "uniqueness"
+
+    def __init__(
+        self,
+        min_count: int | None = None,
+        max_count: int | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.min_count = min_count
+        self.max_count = max_count
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_target_columns(lf)
+
+        if not columns:
+            return issues
+
+        exprs = [
+            pl.len().alias("_total"),
+            *[pl.col(c).n_unique().alias(f"_unique_{c}") for c in columns],
+        ]
+        result = lf.select(exprs).collect()
+
+        for col in columns:
+            unique_count = result[f"_unique_{col}"][0]
+
+            if self.min_count is not None and unique_count < self.min_count:
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="distinct_count_low",
+                        count=self.min_count - unique_count,
+                        severity=Severity.MEDIUM,
+                        details=f"Distinct count {unique_count} < min {self.min_count}",
+                        expected=f">= {self.min_count}",
+                        actual=unique_count,
+                    )
+                )
+
+            if self.max_count is not None and unique_count > self.max_count:
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="distinct_count_high",
+                        count=unique_count - self.max_count,
+                        severity=Severity.MEDIUM,
+                        details=f"Distinct count {unique_count} > max {self.max_count}",
+                        expected=f"<= {self.max_count}",
+                        actual=unique_count,
+                    )
+                )
+
+        return issues