truthound 1.0.8__py3-none-any.whl

truthound/validators/profiling/__init__.py

@@ -0,0 +1,52 @@
+"""Data profiling validators.
+
+This module provides validators for analyzing data quality through
+statistical profiling:
+
+- **Cardinality Validators**: Unique value ratio and count validation
+- **Entropy Validators**: Information entropy and gain analysis
+- **Frequency Validators**: Value frequency distribution validation
+
+Validators:
+    CardinalityValidator: Validates cardinality (uniqueness ratio) bounds
+    UniquenessRatioValidator: Extended uniqueness analysis with duplicates
+    EntropyValidator: Validates Shannon entropy bounds
+    InformationGainValidator: Validates information gain between columns
+    ValueFrequencyValidator: Validates value frequency patterns
+    DistributionShapeValidator: Validates distribution shape (Gini, uniformity)
+"""
+
+from truthound.validators.profiling.base import (
+    ProfileMetrics,
+    ProfilingValidator,
+)
+
+from truthound.validators.profiling.cardinality import (
+    CardinalityValidator,
+    UniquenessRatioValidator,
+)
+
+from truthound.validators.profiling.entropy import (
+    EntropyValidator,
+    InformationGainValidator,
+)
+
+from truthound.validators.profiling.frequency import (
+    ValueFrequencyValidator,
+    DistributionShapeValidator,
+)
+
+__all__ = [
+    # Base classes
+    "ProfileMetrics",
+    "ProfilingValidator",
+    # Cardinality validators
+    "CardinalityValidator",
+    "UniquenessRatioValidator",
+    # Entropy validators
+    "EntropyValidator",
+    "InformationGainValidator",
+    # Frequency validators
+    "ValueFrequencyValidator",
+    "DistributionShapeValidator",
+]

truthound/validators/profiling/base.py

@@ -0,0 +1,175 @@
+"""Base classes for data profiling validators.
+
+This module provides extensible base classes for implementing
+data profiling and quality metrics validators.
+"""
+
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+import polars as pl
+import numpy as np
+
+from truthound.validators.base import (
+    Validator,
+    ValidationIssue,
+)
+
+
+@dataclass
+class ProfileMetrics:
+    """Container for column profiling metrics.
+
+    Attributes:
+        total_count: Total number of rows
+        null_count: Number of null values
+        unique_count: Number of unique values
+        cardinality: Unique count / total count
+        entropy: Shannon entropy
+        most_common: Most common value and count
+        least_common: Least common value and count
+    """
+
+    total_count: int = 0
+    null_count: int = 0
+    unique_count: int = 0
+    cardinality: float = 0.0
+    entropy: float = 0.0
+    most_common: tuple[Any, int] | None = None
+    least_common: tuple[Any, int] | None = None
+
+
+class ProfilingValidator(Validator):
+    """Base class for data profiling validators.
+
+    Data profiling validators analyze column characteristics
+    to detect quality issues based on statistical properties
+    like cardinality, entropy, and value distributions.
+
+    Subclasses should implement:
+        - validate_profile(): Core validation logic
+        - validate(): Full validation method
+    """
+
+    category = "profiling"
+
+    def __init__(
+        self,
+        column: str,
+        include_nulls: bool = False,
+        **kwargs: Any,
+    ):
+        """Initialize profiling validator.
+
+        Args:
+            column: Column to profile
+            include_nulls: Whether to include nulls in calculations
+            **kwargs: Additional config
+        """
+        super().__init__(**kwargs)
+        self.column = column
+        self.include_nulls = include_nulls
+
+    def _compute_value_counts(self, df: pl.DataFrame) -> pl.DataFrame:
+        """Compute value counts for the column.
+
+        Args:
+            df: Input DataFrame
+
+        Returns:
+            DataFrame with value counts
+        """
+        col_data = df[self.column]
+
+        if not self.include_nulls:
+            col_data = col_data.drop_nulls()
+
+        return (
+            pl.DataFrame({self.column: col_data})
+            .group_by(self.column)
+            .agg(pl.len().alias("count"))
+            .sort("count", descending=True)
+        )
+
+    def _compute_entropy(self, value_counts: pl.DataFrame) -> float:
+        """Compute Shannon entropy.
+
+        Args:
+            value_counts: DataFrame with value counts
+
+        Returns:
+            Entropy value in bits
+        """
+        if len(value_counts) == 0:
+            return 0.0
+
+        counts = value_counts["count"].to_numpy()
+        total = counts.sum()
+
+        if total == 0:
+            return 0.0
+
+        probabilities = counts / total
+        # Filter out zero probabilities to avoid log(0)
+        probabilities = probabilities[probabilities > 0]
+
+        entropy = -np.sum(probabilities * np.log2(probabilities))
+        return float(entropy)
+
+    def _compute_metrics(self, df: pl.DataFrame) -> ProfileMetrics:
+        """Compute all profiling metrics.
+
+        Args:
+            df: Input DataFrame
+
+        Returns:
+            ProfileMetrics object
+        """
+        metrics = ProfileMetrics()
+
+        col_data = df[self.column]
+        metrics.total_count = len(col_data)
+        metrics.null_count = col_data.null_count()
+
+        if metrics.total_count == 0:
+            return metrics
+
+        # Compute value counts
+        value_counts = self._compute_value_counts(df)
+
+        if len(value_counts) > 0:
+            metrics.unique_count = len(value_counts)
+
+            # Cardinality
+            non_null_count = metrics.total_count - metrics.null_count
+            if non_null_count > 0:
+                metrics.cardinality = metrics.unique_count / non_null_count
+
+            # Entropy
+            metrics.entropy = self._compute_entropy(value_counts)
+
+            # Most common
+            first_row = value_counts.row(0)
+            metrics.most_common = (first_row[0], first_row[1])
+
+            # Least common
+            last_row = value_counts.row(-1)
+            metrics.least_common = (last_row[0], last_row[1])
+
+        return metrics
+
+    @abstractmethod
+    def validate_profile(
+        self, df: pl.DataFrame, metrics: ProfileMetrics
+    ) -> list[ValidationIssue]:
+        """Validate based on profile metrics.
+
+        Args:
+            df: Input DataFrame
+            metrics: Computed profile metrics
+
+        Returns:
+            List of validation issues
+        """
+        pass

truthound/validators/profiling/cardinality.py

@@ -0,0 +1,312 @@
+"""Cardinality-based profiling validators.
+
+This module provides validators that analyze column cardinality
+(the ratio of unique values to total values).
+"""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue
+from truthound.validators.registry import register_validator
+from truthound.validators.profiling.base import (
+    ProfileMetrics,
+    ProfilingValidator,
+)
+
+
+@register_validator
+class CardinalityValidator(ProfilingValidator):
+    """Validates column cardinality is within expected range.
+
+    Cardinality = unique_count / total_count
+
+    Use cases:
+    - Detect ID columns (cardinality ≈ 1.0)
+    - Detect categorical columns (low cardinality)
+    - Detect potential data issues (unexpected cardinality changes)
+
+    Example:
+        validator = CardinalityValidator(
+            column="user_id",
+            min_cardinality=0.9,  # Expect high uniqueness
+            max_cardinality=1.0,
+        )
+    """
+
+    name = "cardinality"
+
+    def __init__(
+        self,
+        column: str,
+        min_cardinality: float = 0.0,
+        max_cardinality: float = 1.0,
+        min_unique_count: int | None = None,
+        max_unique_count: int | None = None,
+        **kwargs: Any,
+    ):
+        """Initialize cardinality validator.
+
+        Args:
+            column: Column to validate
+            min_cardinality: Minimum cardinality ratio (0-1)
+            max_cardinality: Maximum cardinality ratio (0-1)
+            min_unique_count: Minimum absolute unique count
+            max_unique_count: Maximum absolute unique count
+            **kwargs: Additional config
+        """
+        super().__init__(column=column, **kwargs)
+        self.min_cardinality = min_cardinality
+        self.max_cardinality = max_cardinality
+        self.min_unique_count = min_unique_count
+        self.max_unique_count = max_unique_count
+
+    def validate_profile(
+        self, df: pl.DataFrame, metrics: ProfileMetrics
+    ) -> list[ValidationIssue]:
+        """Validate cardinality constraints.
+
+        Args:
+            df: Input DataFrame
+            metrics: Profile metrics
+
+        Returns:
+            List of validation issues
+        """
+        issues: list[ValidationIssue] = []
+
+        if metrics.total_count == 0:
+            return issues
+
+        # Check cardinality ratio
+        if metrics.cardinality < self.min_cardinality:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="cardinality_too_low",
+                    count=1,
+                    severity=Severity.MEDIUM,
+                    details=(
+                        f"Cardinality ({metrics.cardinality:.4f}) below minimum "
+                        f"({self.min_cardinality:.4f}). "
+                        f"Unique: {metrics.unique_count}, Total: {metrics.total_count}"
+                    ),
+                    expected=f"Cardinality >= {self.min_cardinality}",
+                )
+            )
+
+        if metrics.cardinality > self.max_cardinality:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="cardinality_too_high",
+                    count=1,
+                    severity=Severity.MEDIUM,
+                    details=(
+                        f"Cardinality ({metrics.cardinality:.4f}) above maximum "
+                        f"({self.max_cardinality:.4f}). "
+                        f"Unique: {metrics.unique_count}, Total: {metrics.total_count}"
+                    ),
+                    expected=f"Cardinality <= {self.max_cardinality}",
+                )
+            )
+
+        # Check absolute unique count
+        if self.min_unique_count is not None:
+            if metrics.unique_count < self.min_unique_count:
+                issues.append(
+                    ValidationIssue(
+                        column=self.column,
+                        issue_type="unique_count_too_low",
+                        count=metrics.unique_count,
+                        severity=Severity.MEDIUM,
+                        details=(
+                            f"Unique count ({metrics.unique_count}) below minimum "
+                            f"({self.min_unique_count})."
+                        ),
+                        expected=f"Unique count >= {self.min_unique_count}",
+                    )
+                )
+
+        if self.max_unique_count is not None:
+            if metrics.unique_count > self.max_unique_count:
+                issues.append(
+                    ValidationIssue(
+                        column=self.column,
+                        issue_type="unique_count_too_high",
+                        count=metrics.unique_count,
+                        severity=Severity.MEDIUM,
+                        details=(
+                            f"Unique count ({metrics.unique_count}) above maximum "
+                            f"({self.max_unique_count})."
+                        ),
+                        expected=f"Unique count <= {self.max_unique_count}",
+                    )
+                )
+
+        return issues
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        """Validate the LazyFrame.
+
+        Args:
+            lf: Input LazyFrame
+
+        Returns:
+            List of validation issues
+        """
+        df = lf.select(pl.col(self.column)).collect()
+        metrics = self._compute_metrics(df)
+        return self.validate_profile(df, metrics)
+
+
+@register_validator
+class UniquenessRatioValidator(ProfilingValidator):
+    """Validates uniqueness ratio with detailed analysis.
+
+    Extends cardinality validation with:
+    - Duplicate detection
+    - Top duplicate values
+    - Uniqueness trend analysis
+
+    Example:
+        validator = UniquenessRatioValidator(
+            column="email",
+            min_uniqueness=0.99,
+            report_top_duplicates=5,
+        )
+    """
+
+    name = "uniqueness_ratio"
+
+    def __init__(
+        self,
+        column: str,
+        min_uniqueness: float = 0.0,
+        max_uniqueness: float = 1.0,
+        report_top_duplicates: int = 5,
+        **kwargs: Any,
+    ):
+        """Initialize uniqueness ratio validator.
+
+        Args:
+            column: Column to validate
+            min_uniqueness: Minimum uniqueness ratio
+            max_uniqueness: Maximum uniqueness ratio
+            report_top_duplicates: Number of top duplicates to report
+            **kwargs: Additional config
+        """
+        super().__init__(column=column, **kwargs)
+        self.min_uniqueness = min_uniqueness
+        self.max_uniqueness = max_uniqueness
+        self.report_top_duplicates = report_top_duplicates
+
+    def _get_top_duplicates(
+        self, df: pl.DataFrame, n: int
+    ) -> list[tuple[Any, int]]:
+        """Get top n most duplicated values.
+
+        Args:
+            df: Input DataFrame
+            n: Number of duplicates to return
+
+        Returns:
+            List of (value, count) tuples
+        """
+        value_counts = self._compute_value_counts(df)
+
+        # Filter to only duplicates
+        duplicates = value_counts.filter(pl.col("count") > 1).head(n)
+
+        return [
+            (row[0], row[1])
+            for row in duplicates.iter_rows()
+        ]
+
+    def validate_profile(
+        self, df: pl.DataFrame, metrics: ProfileMetrics
+    ) -> list[ValidationIssue]:
+        """Validate uniqueness ratio.
+
+        Args:
+            df: Input DataFrame
+            metrics: Profile metrics
+
+        Returns:
+            List of validation issues
+        """
+        issues: list[ValidationIssue] = []
+
+        if metrics.total_count == 0:
+            return issues
+
+        uniqueness = metrics.cardinality
+
+        if uniqueness < self.min_uniqueness:
+            # Get top duplicates for details
+            top_dupes = self._get_top_duplicates(df, self.report_top_duplicates)
+            duplicate_count = metrics.total_count - metrics.unique_count
+
+            dupe_details = ""
+            if top_dupes:
+                dupe_strs = [f"{val!r}: {cnt}x" for val, cnt in top_dupes]
+                dupe_details = f" Top duplicates: {', '.join(dupe_strs)}"
+
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="uniqueness_too_low",
+                    count=duplicate_count,
+                    severity=self._calculate_severity(1 - uniqueness),
+                    details=(
+                        f"Uniqueness ratio ({uniqueness:.4f}) below minimum "
+                        f"({self.min_uniqueness:.4f}). "
+                        f"{duplicate_count} duplicate values.{dupe_details}"
+                    ),
+                    expected=f"Uniqueness >= {self.min_uniqueness}",
+                )
+            )
+
+        if uniqueness > self.max_uniqueness:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="uniqueness_too_high",
+                    count=1,
+                    severity=Severity.LOW,
+                    details=(
+                        f"Uniqueness ratio ({uniqueness:.4f}) above maximum "
+                        f"({self.max_uniqueness:.4f}). "
+                        f"This column may be more unique than expected."
+                    ),
+                    expected=f"Uniqueness <= {self.max_uniqueness}",
+                )
+            )
+
+        return issues
+
+    def _calculate_severity(self, duplicate_ratio: float) -> Severity:
+        """Calculate severity based on duplicate ratio."""
+        if duplicate_ratio < 0.01:
+            return Severity.LOW
+        elif duplicate_ratio < 0.05:
+            return Severity.MEDIUM
+        elif duplicate_ratio < 0.1:
+            return Severity.HIGH
+        else:
+            return Severity.CRITICAL
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        """Validate the LazyFrame.
+
+        Args:
+            lf: Input LazyFrame
+
+        Returns:
+            List of validation issues
+        """
+        df = lf.select(pl.col(self.column)).collect()
+        metrics = self._compute_metrics(df)
+        return self.validate_profile(df, metrics)