truthound-dashboard 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

truthound_dashboard/schemas/validators/base.py

@@ -0,0 +1,263 @@
+"""Base validator schema definitions.
+
+This module defines the core types and enums used across all validator categories.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class ValidatorCategory(str, Enum):
+    """Validator categories matching truthound's classification.
+
+    Categories are organized by validation purpose:
+    - Core validators: schema, completeness, uniqueness, distribution
+    - Format validators: string, datetime
+    - Statistical validators: aggregate, anomaly, drift
+    - Relational validators: cross_table, multi_column, query
+    - Domain validators: table, geospatial, privacy, business
+    - Advanced validators: time_series, referential, streaming
+    """
+
+    # Core validators (no extra dependencies)
+    SCHEMA = "schema"
+    COMPLETENESS = "completeness"
+    UNIQUENESS = "uniqueness"
+    DISTRIBUTION = "distribution"
+
+    # Format validators
+    STRING = "string"
+    DATETIME = "datetime"
+
+    # Statistical validators
+    AGGREGATE = "aggregate"
+    ANOMALY = "anomaly"  # requires: anomaly (scipy + sklearn)
+    DRIFT = "drift"  # requires: drift (scipy)
+
+    # Relational validators
+    CROSS_TABLE = "cross_table"
+    MULTI_COLUMN = "multi_column"
+    QUERY = "query"
+
+    # Domain validators
+    TABLE = "table"
+    GEOSPATIAL = "geospatial"
+    PRIVACY = "privacy"
+    BUSINESS = "business"
+
+    # Advanced validators
+    TIME_SERIES = "time_series"
+    REFERENTIAL = "referential"
+    STREAMING = "streaming"
+
+
+class ParameterType(str, Enum):
+    """Supported parameter types for validator configuration."""
+
+    STRING = "string"
+    STRING_LIST = "string_list"
+    INTEGER = "integer"
+    FLOAT = "float"
+    BOOLEAN = "boolean"
+    SELECT = "select"  # Single selection from options
+    MULTI_SELECT = "multi_select"  # Multiple selections
+    COLUMN = "column"  # Column name from data source
+    COLUMN_LIST = "column_list"  # Multiple column names
+    SCHEMA = "schema"  # JSON/YAML schema definition
+    EXPRESSION = "expression"  # Polars expression string
+    REGEX = "regex"  # Regular expression pattern
+    DATE = "date"  # Date value (YYYY-MM-DD)
+    DATETIME = "datetime"  # Datetime value
+    SOURCE_REF = "source_ref"  # Reference to another data source
+
+
+class ParameterDefinition(BaseModel):
+    """Definition of a validator parameter."""
+
+    name: str = Field(..., description="Parameter name (matches truthound API)")
+    label: str = Field(..., description="Display label for UI")
+    type: ParameterType = Field(..., description="Parameter type")
+    description: str = Field(default="", description="Help text for the parameter")
+    required: bool = Field(default=False, description="Whether parameter is required")
+    default: Any = Field(default=None, description="Default value if not specified")
+    options: list[dict[str, str]] | None = Field(
+        default=None,
+        description="Options for select/multi_select types [{value, label}]",
+    )
+    min_value: float | None = Field(default=None, description="Minimum for numeric types")
+    max_value: float | None = Field(default=None, description="Maximum for numeric types")
+    placeholder: str | None = Field(default=None, description="Placeholder text")
+    validation_pattern: str | None = Field(
+        default=None, description="Regex pattern for validation"
+    )
+    depends_on: str | None = Field(
+        default=None,
+        description="Parameter name this depends on (for conditional display)",
+    )
+    depends_value: Any = Field(
+        default=None,
+        description="Value the dependency must have for this param to show",
+    )
+    group: str | None = Field(
+        default=None,
+        description="Parameter group for UI organization",
+    )
+
+
+class ValidatorDefinition(BaseModel):
+    """Complete definition of a validator including its parameters."""
+
+    name: str = Field(..., description="Validator class name (e.g., 'ColumnExists')")
+    display_name: str = Field(..., description="Human-readable name")
+    category: ValidatorCategory = Field(..., description="Validator category")
+    description: str = Field(..., description="What this validator checks")
+    parameters: list[ParameterDefinition] = Field(
+        default_factory=list, description="Configurable parameters"
+    )
+    tags: list[str] = Field(default_factory=list, description="Searchable tags")
+    severity_default: Literal["low", "medium", "high", "critical"] = Field(
+        default="medium", description="Default issue severity"
+    )
+    requires_extra: str | None = Field(
+        default=None,
+        description="Extra dependency required (e.g., 'drift', 'anomaly')",
+    )
+    experimental: bool = Field(
+        default=False,
+        description="Whether this validator is experimental",
+    )
+    deprecated: bool = Field(
+        default=False,
+        description="Whether this validator is deprecated",
+    )
+    deprecation_message: str | None = Field(
+        default=None,
+        description="Message explaining deprecation and alternatives",
+    )
+
+
+class ValidatorConfig(BaseModel):
+    """Configuration for running a specific validator with parameters."""
+
+    name: str = Field(..., description="Validator name")
+    enabled: bool = Field(default=True, description="Whether to run this validator")
+    params: dict[str, Any] = Field(
+        default_factory=dict, description="Parameter values"
+    )
+    severity_override: Literal["low", "medium", "high", "critical"] | None = Field(
+        default=None, description="Override default severity"
+    )
+
+
+class ValidatorConfigList(BaseModel):
+    """List of validator configurations for a validation run."""
+
+    validators: list[ValidatorConfig] = Field(
+        default_factory=list, description="Configured validators"
+    )
+
+
+# ============================================================================
+# Validator Config Conversion Utilities
+# ============================================================================
+
+
+def configs_to_truthound_format(
+    configs: list[ValidatorConfig],
+) -> tuple[list[str] | None, dict[str, dict[str, Any]]]:
+    """Convert ValidatorConfig list to truthound-compatible format.
+
+    truthound supports two ways of specifying validators:
+    1. Simple list of validator names: validators=["Null", "Duplicate"]
+    2. Dict-based configuration: validator_params={"Null": {"columns": ["a", "b"]}}
+
+    This function converts our ValidatorConfig format to both formats,
+    allowing the caller to choose based on whether custom params exist.
+
+    Args:
+        configs: List of ValidatorConfig objects from API request.
+
+    Returns:
+        Tuple of (validator_names, validator_params):
+        - validator_names: List of enabled validator names, or None if empty
+        - validator_params: Dict of {validator_name: {param: value}} for
+          validators with non-default parameters
+
+    Example:
+        >>> configs = [
+        ...     ValidatorConfig(name="Null", enabled=True, params={"columns": ["a"]}),
+        ...     ValidatorConfig(name="Duplicate", enabled=True, params={}),
+        ...     ValidatorConfig(name="Range", enabled=False, params={}),
+        ... ]
+        >>> names, params = configs_to_truthound_format(configs)
+        >>> names
+        ['Null', 'Duplicate']
+        >>> params
+        {'Null': {'columns': ['a']}}
+    """
+    enabled_names: list[str] = []
+    validator_params: dict[str, dict[str, Any]] = {}
+
+    for config in configs:
+        if not config.enabled:
+            continue
+
+        enabled_names.append(config.name)
+
+        # Only include params that are non-empty
+        if config.params:
+            # Filter out None, empty strings, and empty lists
+            filtered_params = {
+                k: v
+                for k, v in config.params.items()
+                if v is not None and v != "" and v != []
+            }
+            if filtered_params:
+                validator_params[config.name] = filtered_params
+
+    return enabled_names if enabled_names else None, validator_params
+
+
+def has_custom_params(configs: list[ValidatorConfig]) -> bool:
+    """Check if any configs have custom (non-default) parameters.
+
+    This helps determine whether to use simple or advanced mode when
+    calling truthound.
+
+    Args:
+        configs: List of ValidatorConfig objects.
+
+    Returns:
+        True if at least one enabled config has non-empty params.
+    """
+    for config in configs:
+        if not config.enabled:
+            continue
+        if config.params:
+            # Check for any non-empty param values
+            for value in config.params.values():
+                if value is not None and value != "" and value != []:
+                    return True
+    return False
+
+
+def merge_severity_overrides(
+    configs: list[ValidatorConfig],
+) -> dict[str, str]:
+    """Extract severity overrides from configs.
+
+    Args:
+        configs: List of ValidatorConfig objects.
+
+    Returns:
+        Dict mapping validator names to their severity overrides.
+    """
+    return {
+        config.name: config.severity_override
+        for config in configs
+        if config.enabled and config.severity_override is not None
+    }

truthound_dashboard/schemas/validators/completeness_validators.py

@@ -0,0 +1,269 @@
+"""Completeness validators.
+
+Validators for detecting missing values, empty data, and placeholder values.
+"""
+
+from .base import (
+    ParameterDefinition,
+    ParameterType,
+    ValidatorCategory,
+    ValidatorDefinition,
+)
+
+COMPLETENESS_VALIDATORS: list[ValidatorDefinition] = [
+    ValidatorDefinition(
+        name="Null",
+        display_name="Null Values",
+        category=ValidatorCategory.COMPLETENESS,
+        description="Detects and reports null values within specified columns.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Target columns (leave empty for all columns)",
+            ),
+            ParameterDefinition(
+                name="mostly",
+                label="Mostly (Threshold)",
+                type=ParameterType.FLOAT,
+                description="Acceptable non-null ratio (0.0-1.0). E.g., 0.95 means 5% nulls allowed.",
+                min_value=0,
+                max_value=1,
+                placeholder="0.95",
+            ),
+        ],
+        tags=["completeness", "null", "missing"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="NotNull",
+        display_name="Not Null",
+        category=ValidatorCategory.COMPLETENESS,
+        description="Ensures the specified column contains no null values.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                description="Column that must not contain nulls",
+                required=True,
+            ),
+        ],
+        tags=["completeness", "null", "required"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="CompletenessRatio",
+        display_name="Completeness Ratio",
+        category=ValidatorCategory.COMPLETENESS,
+        description="Validates that the completeness ratio meets a minimum threshold.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="min_ratio",
+                label="Minimum Ratio",
+                type=ParameterType.FLOAT,
+                description="Minimum completeness ratio (0.0-1.0)",
+                required=True,
+                min_value=0,
+                max_value=1,
+                default=0.95,
+            ),
+        ],
+        tags=["completeness", "ratio", "threshold"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="EmptyString",
+        display_name="Empty String",
+        category=ValidatorCategory.COMPLETENESS,
+        description="Detects empty strings in string columns.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Target string columns (leave empty for all string columns)",
+            ),
+        ],
+        tags=["completeness", "empty", "string"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="WhitespaceOnly",
+        display_name="Whitespace Only",
+        category=ValidatorCategory.COMPLETENESS,
+        description="Identifies values containing only whitespace characters.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Target columns to check",
+            ),
+        ],
+        tags=["completeness", "whitespace", "string"],
+        severity_default="low",
+    ),
+    ValidatorDefinition(
+        name="ConditionalNull",
+        display_name="Conditional Null",
+        category=ValidatorCategory.COMPLETENESS,
+        description="Validates null values based on conditional logic.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                description="Column to check for nulls",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="condition",
+                label="Condition Expression",
+                type=ParameterType.EXPRESSION,
+                description="Polars expression defining when column must not be null",
+                required=True,
+                placeholder='status == "active"',
+            ),
+        ],
+        tags=["completeness", "conditional", "null"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="DefaultValue",
+        display_name="Default Value Detection",
+        category=ValidatorCategory.COMPLETENESS,
+        description="Detects values matching default or placeholder patterns.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="default_values",
+                label="Default Values",
+                type=ParameterType.STRING_LIST,
+                description="Values considered as defaults/placeholders",
+                required=True,
+                placeholder="N/A, TBD, unknown, -1",
+            ),
+        ],
+        tags=["completeness", "default", "placeholder"],
+        severity_default="low",
+    ),
+    ValidatorDefinition(
+        name="RequiredFields",
+        display_name="Required Fields",
+        category=ValidatorCategory.COMPLETENESS,
+        description="Validates that all specified fields have non-null, non-empty values.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Required Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns that must have values",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="treat_empty_as_null",
+                label="Treat Empty as Null",
+                type=ParameterType.BOOLEAN,
+                description="Consider empty strings as missing values",
+                default=True,
+            ),
+        ],
+        tags=["completeness", "required", "mandatory"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="ConditionalRequired",
+        display_name="Conditional Required",
+        category=ValidatorCategory.COMPLETENESS,
+        description="Requires a field when another field meets a condition.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Required Column",
+                type=ParameterType.COLUMN,
+                description="Column that must have a value when condition is met",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="when_column",
+                label="Condition Column",
+                type=ParameterType.COLUMN,
+                description="Column to check for the condition",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="when_value",
+                label="Condition Value",
+                type=ParameterType.STRING,
+                description="Value that triggers the requirement",
+                required=True,
+            ),
+        ],
+        tags=["completeness", "conditional", "required"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="MutualCompleteness",
+        display_name="Mutual Completeness",
+        category=ValidatorCategory.COMPLETENESS,
+        description="Ensures columns are either all null or all non-null together.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns that must have mutual completeness",
+                required=True,
+            ),
+        ],
+        tags=["completeness", "mutual", "coexistence"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="AtLeastOneNotNull",
+        display_name="At Least One Not Null",
+        category=ValidatorCategory.COMPLETENESS,
+        description="Ensures at least one column in a group has a non-null value.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Column group where at least one must be non-null",
+                required=True,
+            ),
+        ],
+        tags=["completeness", "at_least_one", "group"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="ExactlyOneNotNull",
+        display_name="Exactly One Not Null",
+        category=ValidatorCategory.COMPLETENESS,
+        description="Ensures exactly one column in a group has a non-null value.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Column group where exactly one must be non-null",
+                required=True,
+            ),
+        ],
+        tags=["completeness", "exactly_one", "mutex"],
+        severity_default="medium",
+    ),
+]