truthound 1.0.8__py3-none-any.whl

truthound/validators/string/regex.py

@@ -0,0 +1,202 @@
+"""Regex pattern validators.
+
+Provides vectorized regex validation using Polars' native string operations
+for optimal performance on large datasets.
+
+Performance Notes:
+- Uses str.contains() for vectorized regex matching
+- ~100x faster than row-by-row iteration on 1M+ rows
+- Supports both strict (fullmatch) and partial (search) matching
+"""
+
+import re
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import (
+    ValidationIssue,
+    Validator,
+    StringValidatorMixin,
+    RegexValidatorMixin,
+    RegexValidationError,
+)
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class RegexValidator(Validator, StringValidatorMixin, RegexValidatorMixin):
+    """Validates that string values match a regex pattern.
+
+    Uses Polars' native str.contains() for vectorized regex matching,
+    providing ~100x performance improvement over row-by-row iteration.
+
+    Pattern is validated at construction time to catch errors early.
+
+    Example:
+        # Valid pattern
+        validator = RegexValidator(pattern=r"^[A-Z]{3}-\d{4}$")
+
+        # Invalid pattern raises immediately
+        try:
+            validator = RegexValidator(pattern=r"[invalid(")
+        except RegexValidationError as e:
+            print(f"Bad pattern: {e}")
+
+    Performance:
+        - 1M rows: ~50ms (vectorized) vs ~5000ms (iteration)
+    """
+
+    name = "regex"
+    category = "string"
+
+    def __init__(
+        self,
+        pattern: str,
+        match_full: bool = True,
+        dotall: bool = True,
+        case_insensitive: bool = False,
+        **kwargs: Any,
+    ):
+        """Initialize with pattern validation.
+
+        Args:
+            pattern: Regex pattern string (validated at construction)
+            match_full: If True, pattern must match entire string (adds ^ and $ anchors)
+            dotall: If True, . matches newlines (note: Polars uses (?s) flag)
+            case_insensitive: If True, ignore case (note: Polars uses (?i) flag)
+            **kwargs: Additional validator config
+
+        Raises:
+            RegexValidationError: If pattern is invalid
+        """
+        super().__init__(**kwargs)
+
+        # Build flags for Python regex validation
+        flags = 0
+        if dotall:
+            flags |= re.DOTALL
+        if case_insensitive:
+            flags |= re.IGNORECASE
+
+        # Validate pattern at construction time using Python regex
+        self._compiled = self.validate_pattern(pattern, flags)
+        self.pattern = pattern
+        self.match_full = match_full
+        self.case_insensitive = case_insensitive
+        self.dotall = dotall
+
+        # Build Polars-compatible pattern string
+        self._polars_pattern = self._build_polars_pattern()
+
+    def _build_polars_pattern(self) -> str:
+        """Build pattern string for Polars str.contains().
+
+        Polars uses inline flags (?i), (?s), etc. instead of Python's re.IGNORECASE.
+        For full match, we ensure the pattern has ^ and $ anchors.
+        """
+        pattern = self.pattern
+
+        # Add anchors for full match if not already present
+        if self.match_full:
+            if not pattern.startswith("^"):
+                pattern = "^" + pattern
+            if not pattern.endswith("$"):
+                pattern = pattern + "$"
+
+        # Add inline flags for Polars
+        flags = ""
+        if self.case_insensitive:
+            flags += "i"
+        if self.dotall:
+            flags += "s"
+
+        if flags:
+            pattern = f"(?{flags}){pattern}"
+
+        return pattern
+
+    def _build_match_expr(self, col: str) -> pl.Expr:
+        """Build vectorized match expression for a column.
+
+        Returns expression that is True for INVALID values.
+        """
+        col_expr = pl.col(col)
+
+        # Skip null and empty strings
+        non_empty = col_expr.is_not_null() & (col_expr.str.strip_chars() != "")
+
+        # Use Polars native regex matching
+        matches = col_expr.str.contains(self._polars_pattern)
+
+        # Return True for invalid (non-empty but not matching)
+        return non_empty & ~matches
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_string_columns(lf)
+
+        if not columns:
+            return issues
+
+        # Build expressions for all columns in a single query
+        exprs: list[pl.Expr] = [pl.len().alias("_total")]
+
+        for col in columns:
+            invalid_expr = self._build_match_expr(col)
+            exprs.append(invalid_expr.sum().alias(f"_inv_{col}"))
+            exprs.append(pl.col(col).is_not_null().sum().alias(f"_nn_{col}"))
+
+        result = lf.select(exprs).collect()
+        total_rows = result["_total"][0]
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            invalid_count = result[f"_inv_{col}"][0]
+            non_null_count = result[f"_nn_{col}"][0]
+
+            if invalid_count > 0 and non_null_count > 0:
+                # Check mostly threshold
+                if self._passes_mostly(invalid_count, non_null_count):
+                    continue
+
+                ratio = invalid_count / non_null_count
+
+                # Get sample invalid values
+                samples = self._get_invalid_samples(lf, col)
+
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="regex_mismatch",
+                        count=invalid_count,
+                        severity=self._calculate_severity(ratio),
+                        details=f"Pattern: {self.pattern}",
+                        expected=self.pattern,
+                        sample_values=samples,
+                    )
+                )
+
+        return issues
+
+    def _get_invalid_samples(self, lf: pl.LazyFrame, col: str) -> list[str]:
+        """Get sample invalid values for error reporting."""
+        invalid_expr = self._build_match_expr(col)
+
+        sample_df = (
+            lf.filter(invalid_expr)
+            .select(col)
+            .head(self.config.sample_size)
+            .collect()
+        )
+
+        samples = []
+        for val in sample_df.get_column(col).to_list():
+            if val is not None:
+                display = val[:50] + "..." if len(str(val)) > 50 else str(val)
+                samples.append(display)
+
+        return samples

truthound/validators/string/regex_extended.py

@@ -0,0 +1,435 @@
+"""Extended regex validators for complex pattern matching.
+
+Provides vectorized regex validation using Polars' native string operations
+for optimal performance on large datasets.
+
+Performance Notes:
+- Uses str.contains() for vectorized regex matching
+- ~100x faster than row-by-row iteration on 1M+ rows
+"""
+
+import re
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import (
+    ValidationIssue,
+    Validator,
+    StringValidatorMixin,
+    RegexValidatorMixin,
+    RegexValidationError,
+)
+from truthound.validators.registry import register_validator
+
+
+def _build_polars_pattern(pattern: str, match_full: bool = True) -> str:
+    """Build Polars-compatible pattern with anchors if needed."""
+    if match_full:
+        if not pattern.startswith("^"):
+            pattern = "^" + pattern
+        if not pattern.endswith("$"):
+            pattern = pattern + "$"
+    return pattern
+
+
+@register_validator
+class RegexListValidator(Validator, StringValidatorMixin, RegexValidatorMixin):
+    """Validates that values match at least one pattern from a list.
+
+    Uses Polars' native str.contains() for vectorized regex matching,
+    providing ~100x performance improvement over row-by-row iteration.
+
+    All patterns are validated at construction time to catch errors early.
+
+    Example:
+        # Value should match at least one date format
+        validator = RegexListValidator(
+            patterns=[
+                r"\\d{4}-\\d{2}-\\d{2}",  # YYYY-MM-DD
+                r"\\d{2}/\\d{2}/\\d{4}",  # DD/MM/YYYY
+                r"\\d{2}\\.\\d{2}\\.\\d{4}",  # DD.MM.YYYY
+            ],
+            match_mode="any",  # Match any pattern
+        )
+
+    Raises:
+        RegexValidationError: If any pattern is invalid
+
+    Performance:
+        - 1M rows: ~50ms (vectorized) vs ~5000ms (iteration)
+    """
+
+    name = "regex_list"
+    category = "string"
+
+    def __init__(
+        self,
+        patterns: list[str],
+        match_mode: str = "any",  # "any" = match at least one, "all" = match all
+        match_full: bool = True,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.patterns = patterns
+        self.match_mode = match_mode
+        self.match_full = match_full
+        # Validate all patterns at construction time
+        self._compiled = self.validate_patterns(patterns, re.DOTALL)
+        # Build Polars-compatible patterns
+        self._polars_patterns = [
+            _build_polars_pattern(p, match_full) for p in patterns
+        ]
+
+    def _build_match_expr(self, col: str) -> pl.Expr:
+        """Build vectorized match expression for a column.
+
+        Returns expression that is True for INVALID values.
+        """
+        col_expr = pl.col(col)
+
+        # Skip null and empty strings
+        non_empty = col_expr.is_not_null() & (col_expr.str.strip_chars() != "")
+
+        if self.match_mode == "any":
+            # At least one pattern must match
+            # Invalid = non_empty AND NOT (match_p1 OR match_p2 OR ...)
+            any_match = pl.lit(False)
+            for pattern in self._polars_patterns:
+                any_match = any_match | col_expr.str.contains(pattern)
+            return non_empty & ~any_match
+        else:  # match_mode == "all"
+            # All patterns must match
+            # Invalid = non_empty AND NOT (match_p1 AND match_p2 AND ...)
+            all_match = pl.lit(True)
+            for pattern in self._polars_patterns:
+                all_match = all_match & col_expr.str.contains(pattern)
+            return non_empty & ~all_match
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_string_columns(lf)
+
+        if not columns:
+            return issues
+
+        # Build expressions for all columns in a single query
+        exprs: list[pl.Expr] = [pl.len().alias("_total")]
+
+        for col in columns:
+            invalid_expr = self._build_match_expr(col)
+            exprs.append(invalid_expr.sum().alias(f"_inv_{col}"))
+            exprs.append(pl.col(col).is_not_null().sum().alias(f"_nn_{col}"))
+
+        result = lf.select(exprs).collect()
+        total_rows = result["_total"][0]
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            invalid_count = result[f"_inv_{col}"][0]
+            non_null_count = result[f"_nn_{col}"][0]
+
+            if invalid_count > 0 and non_null_count > 0:
+                # Check mostly threshold
+                if self._passes_mostly(invalid_count, non_null_count):
+                    continue
+
+                ratio = invalid_count / non_null_count
+
+                # Get sample invalid values
+                samples = self._get_invalid_samples(lf, col)
+
+                mode_desc = "any of" if self.match_mode == "any" else "all of"
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="regex_list_mismatch",
+                        count=invalid_count,
+                        severity=self._calculate_severity(ratio),
+                        details=f"Values must match {mode_desc}: {self.patterns[:3]}{'...' if len(self.patterns) > 3 else ''}",
+                        expected=f"Match {mode_desc} {len(self.patterns)} patterns",
+                        sample_values=samples,
+                    )
+                )
+
+        return issues
+
+    def _get_invalid_samples(self, lf: pl.LazyFrame, col: str) -> list[str]:
+        """Get sample invalid values for error reporting."""
+        invalid_expr = self._build_match_expr(col)
+
+        sample_df = (
+            lf.filter(invalid_expr)
+            .select(col)
+            .head(self.config.sample_size)
+            .collect()
+        )
+
+        samples = []
+        for val in sample_df.get_column(col).to_list():
+            if val is not None:
+                display = val[:50] + "..." if len(str(val)) > 50 else str(val)
+                samples.append(display)
+
+        return samples
+
+
+@register_validator
+class NotMatchRegexValidator(Validator, StringValidatorMixin, RegexValidatorMixin):
+    """Validates that values do NOT match a regex pattern.
+
+    Uses Polars' native str.contains() for vectorized regex matching,
+    providing ~100x performance improvement over row-by-row iteration.
+
+    Pattern is validated at construction time to catch errors early.
+
+    Example:
+        # Values should not contain PII patterns
+        validator = NotMatchRegexValidator(
+            pattern=r"\\d{3}-\\d{2}-\\d{4}",  # SSN pattern
+        )
+
+    Raises:
+        RegexValidationError: If pattern is invalid
+
+    Performance:
+        - 1M rows: ~50ms (vectorized) vs ~5000ms (iteration)
+    """
+
+    name = "not_match_regex"
+    category = "string"
+
+    def __init__(
+        self,
+        pattern: str,
+        match_full: bool = False,  # Default to search mode for "not match"
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.pattern = pattern
+        self.match_full = match_full
+        # Validate pattern at construction time
+        self._compiled = self.validate_pattern(pattern, re.DOTALL)
+        # Build Polars-compatible pattern
+        self._polars_pattern = _build_polars_pattern(pattern, match_full)
+
+    def _build_match_expr(self, col: str) -> pl.Expr:
+        """Build vectorized match expression for a column.
+
+        Returns expression that is True for values that MATCH (which is invalid).
+        """
+        col_expr = pl.col(col)
+
+        # Skip null and empty strings
+        non_empty = col_expr.is_not_null() & (col_expr.str.strip_chars() != "")
+
+        # Values that match the pattern are invalid (should NOT match)
+        matches = col_expr.str.contains(self._polars_pattern)
+
+        return non_empty & matches
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_string_columns(lf)
+
+        if not columns:
+            return issues
+
+        # Build expressions for all columns in a single query
+        exprs: list[pl.Expr] = [pl.len().alias("_total")]
+
+        for col in columns:
+            matching_expr = self._build_match_expr(col)
+            exprs.append(matching_expr.sum().alias(f"_match_{col}"))
+            exprs.append(pl.col(col).is_not_null().sum().alias(f"_nn_{col}"))
+
+        result = lf.select(exprs).collect()
+        total_rows = result["_total"][0]
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            matching_count = result[f"_match_{col}"][0]
+            non_null_count = result[f"_nn_{col}"][0]
+
+            if matching_count > 0 and non_null_count > 0:
+                # Check mostly threshold
+                if self._passes_mostly(matching_count, non_null_count):
+                    continue
+
+                ratio = matching_count / non_null_count
+
+                # Get sample matching values
+                samples = self._get_matching_samples(lf, col)
+
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="regex_unexpected_match",
+                        count=matching_count,
+                        severity=self._calculate_severity(ratio),
+                        details=f"Values should NOT match: {self.pattern}",
+                        expected=f"No match for {self.pattern}",
+                        sample_values=samples,
+                    )
+                )
+
+        return issues
+
+    def _get_matching_samples(self, lf: pl.LazyFrame, col: str) -> list[str]:
+        """Get sample matching values for error reporting."""
+        matching_expr = self._build_match_expr(col)
+
+        sample_df = (
+            lf.filter(matching_expr)
+            .select(col)
+            .head(self.config.sample_size)
+            .collect()
+        )
+
+        samples = []
+        for val in sample_df.get_column(col).to_list():
+            if val is not None:
+                display = val[:50] + "..." if len(str(val)) > 50 else str(val)
+                samples.append(display)
+
+        return samples
+
+
+@register_validator
+class NotMatchRegexListValidator(Validator, StringValidatorMixin, RegexValidatorMixin):
+    """Validates that values do NOT match any pattern from a list.
+
+    Uses Polars' native str.contains() for vectorized regex matching,
+    providing ~100x performance improvement over row-by-row iteration.
+
+    All patterns are validated at construction time to catch errors early.
+
+    Example:
+        # Values should not contain any PII patterns
+        validator = NotMatchRegexListValidator(
+            patterns=[
+                r"\\d{3}-\\d{2}-\\d{4}",  # SSN
+                r"\\d{16}",  # Credit card
+                r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}",  # Email
+            ],
+        )
+
+    Raises:
+        RegexValidationError: If any pattern is invalid
+
+    Performance:
+        - 1M rows: ~50ms (vectorized) vs ~5000ms (iteration)
+    """
+
+    name = "not_match_regex_list"
+    category = "string"
+
+    def __init__(
+        self,
+        patterns: list[str],
+        match_full: bool = False,
+        case_insensitive: bool = True,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.patterns = patterns
+        self.match_full = match_full
+        self.case_insensitive = case_insensitive
+        # Validate all patterns at construction time
+        self._compiled = self.validate_patterns(patterns, re.DOTALL | re.IGNORECASE)
+        # Build Polars-compatible patterns
+        self._polars_patterns = []
+        for p in patterns:
+            polars_p = _build_polars_pattern(p, match_full)
+            if case_insensitive:
+                polars_p = f"(?i){polars_p}"
+            self._polars_patterns.append(polars_p)
+
+    def _build_match_expr(self, col: str) -> pl.Expr:
+        """Build vectorized match expression for a column.
+
+        Returns expression that is True for values that MATCH any pattern (which is invalid).
+        """
+        col_expr = pl.col(col)
+
+        # Skip null and empty strings
+        non_empty = col_expr.is_not_null() & (col_expr.str.strip_chars() != "")
+
+        # Values that match ANY forbidden pattern are invalid
+        any_match = pl.lit(False)
+        for pattern in self._polars_patterns:
+            any_match = any_match | col_expr.str.contains(pattern)
+
+        return non_empty & any_match
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_string_columns(lf)
+
+        if not columns:
+            return issues
+
+        # Build expressions for all columns in a single query
+        exprs: list[pl.Expr] = [pl.len().alias("_total")]
+
+        for col in columns:
+            matching_expr = self._build_match_expr(col)
+            exprs.append(matching_expr.sum().alias(f"_match_{col}"))
+            exprs.append(pl.col(col).is_not_null().sum().alias(f"_nn_{col}"))
+
+        result = lf.select(exprs).collect()
+        total_rows = result["_total"][0]
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            matching_count = result[f"_match_{col}"][0]
+            non_null_count = result[f"_nn_{col}"][0]
+
+            if matching_count > 0 and non_null_count > 0:
+                if self._passes_mostly(matching_count, non_null_count):
+                    continue
+
+                ratio = matching_count / non_null_count
+
+                # Get sample matching values
+                samples = self._get_matching_samples(lf, col)
+
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="regex_list_unexpected_match",
+                        count=matching_count,
+                        severity=self._calculate_severity(ratio),
+                        details="Values should NOT match any forbidden pattern",
+                        expected=f"No match for any of {len(self.patterns)} patterns",
+                        sample_values=samples,
+                    )
+                )
+
+        return issues
+
+    def _get_matching_samples(self, lf: pl.LazyFrame, col: str) -> list[str]:
+        """Get sample matching values for error reporting."""
+        matching_expr = self._build_match_expr(col)
+
+        sample_df = (
+            lf.filter(matching_expr)
+            .select(col)
+            .head(self.config.sample_size)
+            .collect()
+        )
+
+        samples = []
+        for val in sample_df.get_column(col).to_list():
+            if val is not None:
+                display = val[:50] + "..." if len(str(val)) > 50 else str(val)
+                samples.append(display)
+
+        return samples