truthound 1.0.8__py3-none-any.whl

truthound/validators/datetime/format.py

@@ -0,0 +1,78 @@
+"""Date format validators."""
+
+from datetime import datetime
+from typing import Any
+
+import polars as pl
+
+from truthound.validators.base import (
+    ValidationIssue,
+    Validator,
+    StringValidatorMixin,
+)
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class DateFormatValidator(Validator, StringValidatorMixin):
+    """Validates that string values match a date format (strftime)."""
+
+    name = "date_format"
+    category = "datetime"
+
+    def __init__(
+        self,
+        format: str = "%Y-%m-%d",
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.format = format
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        columns = self._get_string_columns(lf)
+
+        if not columns:
+            return issues
+
+        df = lf.collect()
+        total_rows = len(df)
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            col_data = df.get_column(col).drop_nulls()
+
+            if len(col_data) == 0:
+                continue
+
+            invalid_count = 0
+            samples = []
+
+            for val in col_data.to_list():
+                if not isinstance(val, str):
+                    continue
+
+                try:
+                    datetime.strptime(val, self.format)
+                except ValueError:
+                    invalid_count += 1
+                    if len(samples) < self.config.sample_size:
+                        samples.append(val)
+
+            if invalid_count > 0:
+                ratio = invalid_count / len(col_data)
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="invalid_date_format",
+                        count=invalid_count,
+                        severity=self._calculate_severity(ratio),
+                        details=f"Expected format: {self.format}",
+                        expected=self.format,
+                        sample_values=samples,
+                    )
+                )
+
+        return issues

truthound/validators/datetime/freshness.py

@@ -0,0 +1,269 @@
+"""Data freshness and temporal coverage validators."""
+
+from datetime import datetime, timedelta, date
+from typing import Any, Literal
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator, DatetimeValidatorMixin
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class RecentDataValidator(Validator, DatetimeValidatorMixin):
+    """Validates that data contains recent records.
+
+    Example:
+        # Data should have records from the last 24 hours
+        validator = RecentDataValidator(
+            column="created_at",
+            max_age_hours=24,
+        )
+    """
+
+    name = "recent_data"
+    category = "datetime"
+
+    def __init__(
+        self,
+        column: str,
+        max_age_hours: float | None = None,
+        max_age_days: float | None = None,
+        reference_time: datetime | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column = column
+        self.reference_time = reference_time or datetime.now()
+
+        if max_age_hours:
+            self.max_age = timedelta(hours=max_age_hours)
+        elif max_age_days:
+            self.max_age = timedelta(days=max_age_days)
+        else:
+            self.max_age = timedelta(hours=24)  # Default 24 hours
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        result = lf.select([
+            pl.col(self.column).max().alias("_max_date"),
+        ]).collect()
+
+        max_date = result["_max_date"][0]
+
+        if max_date is None:
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="no_recent_data",
+                    count=1,
+                    severity=Severity.CRITICAL,
+                    details="No data found in datetime column",
+                )
+            )
+            return issues
+
+        # Convert to datetime if needed
+        if isinstance(max_date, date) and not isinstance(max_date, datetime):
+            max_date = datetime.combine(max_date, datetime.min.time())
+
+        cutoff = self.reference_time - self.max_age
+        age = self.reference_time - max_date
+
+        if max_date < cutoff:
+            hours_old = age.total_seconds() / 3600
+            severity = Severity.CRITICAL if hours_old > self.max_age.total_seconds() / 3600 * 2 else Severity.HIGH
+
+            issues.append(
+                ValidationIssue(
+                    column=self.column,
+                    issue_type="stale_data",
+                    count=1,
+                    severity=severity,
+                    details=f"Most recent data is {hours_old:.1f} hours old",
+                    expected=f"Data within last {self.max_age.total_seconds() / 3600:.1f} hours",
+                    actual=f"Latest: {max_date}",
+                )
+            )
+
+        return issues
+
+
+@register_validator
+class DatePartCoverageValidator(Validator, DatetimeValidatorMixin):
+    """Validates that data covers expected date parts (no gaps).
+
+    Example:
+        # Should have data for every day in the range
+        validator = DatePartCoverageValidator(
+            column="date",
+            date_part="day",
+            min_coverage=0.95,  # Allow 5% missing days
+        )
+    """
+
+    name = "date_part_coverage"
+    category = "datetime"
+
+    def __init__(
+        self,
+        column: str,
+        date_part: Literal["day", "week", "month", "hour"] = "day",
+        min_coverage: float = 1.0,
+        start_date: date | datetime | None = None,
+        end_date: date | datetime | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.column = column
+        self.date_part = date_part
+        self.min_coverage = min_coverage
+        self.start_date = start_date
+        self.end_date = end_date
+
+    def _count_expected_parts(self, start: datetime, end: datetime) -> int:
+        """Count expected date parts in range."""
+        diff = end - start
+
+        if self.date_part == "hour":
+            return int(diff.total_seconds() / 3600) + 1
+        elif self.date_part == "day":
+            return diff.days + 1
+        elif self.date_part == "week":
+            return diff.days // 7 + 1
+        elif self.date_part == "month":
+            return (end.year - start.year) * 12 + (end.month - start.month) + 1
+
+        return diff.days + 1
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        # Get date range from data
+        result = lf.select([
+            pl.col(self.column).min().alias("_min"),
+            pl.col(self.column).max().alias("_max"),
+        ]).collect()
+
+        min_date = result["_min"][0]
+        max_date = result["_max"][0]
+
+        if min_date is None or max_date is None:
+            return issues
+
+        # Use provided dates or data range
+        start = self.start_date or min_date
+        end = self.end_date or max_date
+
+        # Convert to datetime
+        if isinstance(start, date) and not isinstance(start, datetime):
+            start = datetime.combine(start, datetime.min.time())
+        if isinstance(end, date) and not isinstance(end, datetime):
+            end = datetime.combine(end, datetime.max.time())
+
+        # Count distinct date parts
+        if self.date_part == "day":
+            truncate_expr = pl.col(self.column).dt.truncate("1d")
+        elif self.date_part == "hour":
+            truncate_expr = pl.col(self.column).dt.truncate("1h")
+        elif self.date_part == "week":
+            truncate_expr = pl.col(self.column).dt.truncate("1w")
+        elif self.date_part == "month":
+            truncate_expr = pl.col(self.column).dt.truncate("1mo")
+        else:
+            truncate_expr = pl.col(self.column).dt.truncate("1d")
+
+        distinct_parts = lf.select(
+            truncate_expr.n_unique().alias("_distinct")
+        ).collect()["_distinct"][0]
+
+        expected_parts = self._count_expected_parts(start, end)
+
+        if expected_parts > 0:
+            coverage = distinct_parts / expected_parts
+            missing = expected_parts - distinct_parts
+
+            if coverage < self.min_coverage:
+                issues.append(
+                    ValidationIssue(
+                        column=self.column,
+                        issue_type="date_part_coverage_gap",
+                        count=missing,
+                        severity=Severity.HIGH if coverage < 0.8 else Severity.MEDIUM,
+                        details=f"Coverage {coverage:.1%} < required {self.min_coverage:.1%}",
+                        expected=f"{expected_parts} {self.date_part}s",
+                        actual=f"{distinct_parts} {self.date_part}s ({missing} missing)",
+                    )
+                )
+
+        return issues
+
+
+@register_validator
+class GroupedRecentDataValidator(Validator, DatetimeValidatorMixin):
+    """Validates that each group has recent data.
+
+    Example:
+        # Each store should have data from the last 24 hours
+        validator = GroupedRecentDataValidator(
+            datetime_column="transaction_time",
+            group_column="store_id",
+            max_age_hours=24,
+        )
+    """
+
+    name = "grouped_recent_data"
+    category = "datetime"
+
+    def __init__(
+        self,
+        datetime_column: str,
+        group_column: str,
+        max_age_hours: float = 24,
+        reference_time: datetime | None = None,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.datetime_column = datetime_column
+        self.group_column = group_column
+        self.max_age_hours = max_age_hours
+        self.reference_time = reference_time or datetime.now()
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        cutoff = self.reference_time - timedelta(hours=self.max_age_hours)
+
+        # Find groups with stale data
+        group_max = (
+            lf.group_by(self.group_column)
+            .agg(pl.col(self.datetime_column).max().alias("_max_date"))
+            .collect()
+        )
+
+        stale_groups = []
+        for row in group_max.iter_rows(named=True):
+            max_date = row["_max_date"]
+            if max_date is None:
+                stale_groups.append(row[self.group_column])
+            elif isinstance(max_date, date) and not isinstance(max_date, datetime):
+                if datetime.combine(max_date, datetime.min.time()) < cutoff:
+                    stale_groups.append(row[self.group_column])
+            elif max_date < cutoff:
+                stale_groups.append(row[self.group_column])
+
+        if stale_groups:
+            issues.append(
+                ValidationIssue(
+                    column=f"{self.group_column}, {self.datetime_column}",
+                    issue_type="grouped_stale_data",
+                    count=len(stale_groups),
+                    severity=Severity.HIGH,
+                    details=f"{len(stale_groups)} groups have stale data (>{self.max_age_hours}h old)",
+                    sample_values=stale_groups[: self.config.sample_size],
+                )
+            )
+
+        return issues

truthound/validators/datetime/order.py

@@ -0,0 +1,73 @@
+"""Date order validators."""
+
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class DateOrderValidator(Validator):
+    """Validates that one date column is before/after another.
+
+    Example:
+        # start_date should be before end_date
+        validator = DateOrderValidator(
+            first_column="start_date",
+            second_column="end_date",
+            allow_equal=True,
+        )
+    """
+
+    name = "date_order"
+    category = "datetime"
+
+    def __init__(
+        self,
+        first_column: str,
+        second_column: str,
+        allow_equal: bool = True,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.first_column = first_column
+        self.second_column = second_column
+        self.allow_equal = allow_equal
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        if self.allow_equal:
+            violation_expr = pl.col(self.first_column) > pl.col(self.second_column)
+        else:
+            violation_expr = pl.col(self.first_column) >= pl.col(self.second_column)
+
+        result = lf.select([
+            pl.len().alias("_total"),
+            (
+                violation_expr
+                & pl.col(self.first_column).is_not_null()
+                & pl.col(self.second_column).is_not_null()
+            ).sum().alias("_violations"),
+        ]).collect()
+
+        total_rows = result["_total"][0]
+        violations = result["_violations"][0]
+
+        if violations > 0:
+            op = "<=" if self.allow_equal else "<"
+            issues.append(
+                ValidationIssue(
+                    column=f"{self.first_column}, {self.second_column}",
+                    issue_type="date_order_violation",
+                    count=violations,
+                    severity=Severity.HIGH,
+                    details=f"{self.first_column} should be {op} {self.second_column}",
+                    expected=f"{self.first_column} {op} {self.second_column}",
+                )
+            )
+
+        return issues

truthound/validators/datetime/parseable.py

@@ -0,0 +1,185 @@
+"""Date parsing validators."""
+
+from datetime import datetime
+from typing import Any
+
+import polars as pl
+
+from truthound.types import Severity
+from truthound.validators.base import ValidationIssue, Validator, StringValidatorMixin
+from truthound.validators.registry import register_validator
+
+
+@register_validator
+class DateutilParseableValidator(Validator, StringValidatorMixin):
+    """Validates that string values can be parsed as dates using flexible parsing.
+
+    Supports various date formats without requiring explicit format specification.
+    Uses dateutil library if available, otherwise falls back to common formats.
+
+    Example:
+        validator = DateutilParseableValidator(column="date_string")
+
+        # With dateutil, these all parse correctly:
+        # "2024-01-15"
+        # "January 15, 2024"
+        # "15/01/2024"
+        # "Jan 15 2024"
+    """
+
+    name = "dateutil_parseable"
+    category = "datetime"
+
+    # Common date formats to try if dateutil is not available
+    # Ordered by specificity: more specific formats first
+    COMMON_FORMATS = [
+        # ISO formats
+        "%Y-%m-%d",
+        "%Y/%m/%d",
+        "%Y-%m-%dT%H:%M:%S",
+        "%Y-%m-%dT%H:%M:%SZ",
+        "%Y-%m-%dT%H:%M:%S.%f",
+        "%Y-%m-%dT%H:%M:%S.%fZ",
+        "%Y-%m-%d %H:%M:%S",
+        "%Y/%m/%d %H:%M:%S",
+        # Day-Month-Year formats
+        "%d-%m-%Y",
+        "%d/%m/%Y",
+        "%d-%m-%Y %H:%M:%S",
+        "%d/%m/%Y %H:%M:%S",
+        # Month-Day-Year formats (US style)
+        "%m-%d-%Y",
+        "%m/%d/%Y",
+        "%m-%d-%Y %H:%M:%S",
+        "%m/%d/%Y %H:%M:%S",
+        # Named month formats - with comma
+        "%B %d, %Y",              # January 15, 2024
+        "%b %d, %Y",              # Jan 15, 2024
+        "%d %B, %Y",              # 15 January, 2024
+        "%d %b, %Y",              # 15 Jan, 2024
+        # Named month formats - without comma
+        "%B %d %Y",               # January 15 2024
+        "%b %d %Y",               # Jan 15 2024
+        "%d %B %Y",               # 15 January 2024
+        "%d %b %Y",               # 15 Jan 2024
+        # Named month with hyphen
+        "%d-%b-%Y",               # 15-Jan-2024
+        "%d-%B-%Y",               # 15-January-2024
+        "%b-%d-%Y",               # Jan-15-2024
+        "%B-%d-%Y",               # January-15-2024
+        # Named month with time - 12-hour format
+        "%b %d %Y %I:%M %p",      # Jan 15 2024 10:30 AM
+        "%B %d %Y %I:%M %p",      # January 15 2024 10:30 AM
+        "%b %d, %Y %I:%M %p",     # Jan 15, 2024 10:30 AM
+        "%B %d, %Y %I:%M %p",     # January 15, 2024 10:30 AM
+        "%d %b %Y %I:%M %p",      # 15 Jan 2024 10:30 AM
+        "%d %B %Y %I:%M %p",      # 15 January 2024 10:30 AM
+        # Named month with time - 24-hour format
+        "%b %d %Y %H:%M",         # Jan 15 2024 10:30
+        "%B %d %Y %H:%M",         # January 15 2024 10:30
+        "%b %d %Y %H:%M:%S",      # Jan 15 2024 10:30:00
+        "%B %d %Y %H:%M:%S",      # January 15 2024 10:30:00
+        # Other common formats
+        "%Y%m%d",                 # 20240115
+        "%d.%m.%Y",               # 15.01.2024
+        "%Y.%m.%d",               # 2024.01.15
+    ]
+
+    def __init__(
+        self,
+        column: str | None = None,
+        fuzzy: bool = True,
+        dayfirst: bool = False,
+        yearfirst: bool = True,
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.target_column = column
+        self.fuzzy = fuzzy
+        self.dayfirst = dayfirst
+        self.yearfirst = yearfirst
+
+        # Try to import dateutil
+        self._parser = None
+        try:
+            from dateutil import parser
+            self._parser = parser
+        except ImportError:
+            pass
+
+    def _try_parse(self, value: str) -> bool:
+        """Try to parse a date string."""
+        if self._parser:
+            try:
+                self._parser.parse(
+                    value,
+                    fuzzy=self.fuzzy,
+                    dayfirst=self.dayfirst,
+                    yearfirst=self.yearfirst,
+                )
+                return True
+            except (ValueError, TypeError):
+                return False
+        else:
+            # Fallback: try common formats
+            for fmt in self.COMMON_FORMATS:
+                try:
+                    datetime.strptime(value.strip(), fmt)
+                    return True
+                except ValueError:
+                    continue
+            return False
+
+    def validate(self, lf: pl.LazyFrame) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        if self.target_column:
+            columns = [self.target_column]
+        else:
+            columns = self._get_string_columns(lf)
+
+        if not columns:
+            return issues
+
+        df = lf.collect()
+        total_rows = len(df)
+
+        if total_rows == 0:
+            return issues
+
+        for col in columns:
+            col_data = df.get_column(col).drop_nulls()
+
+            if len(col_data) == 0:
+                continue
+
+            unparseable_count = 0
+            samples = []
+
+            for val in col_data.to_list():
+                if not isinstance(val, str):
+                    continue
+
+                if not self._try_parse(val):
+                    unparseable_count += 1
+                    if len(samples) < self.config.sample_size:
+                        samples.append(val[:50] + "..." if len(val) > 50 else val)
+
+            if unparseable_count > 0:
+                if self._passes_mostly(unparseable_count, len(col_data)):
+                    continue
+
+                ratio = unparseable_count / len(col_data)
+                parser_info = "dateutil" if self._parser else "standard formats"
+                issues.append(
+                    ValidationIssue(
+                        column=col,
+                        issue_type="date_not_parseable",
+                        count=unparseable_count,
+                        severity=self._calculate_severity(ratio),
+                        details=f"Values cannot be parsed as dates (using {parser_info})",
+                        sample_values=samples,
+                    )
+                )
+
+        return issues