truthound-dashboard 1.4.4__py3-none-any.whl → 1.5.1__py3-none-any.whl

truthound_dashboard/schemas/validators/profiling_validators.py

@@ -0,0 +1,275 @@
+"""Profiling validators.
+
+Validators for data profiling metrics including cardinality, entropy,
+and value frequency distributions.
+
+Import path in truthound: `from truthound.validators.profiling import *`
+"""
+
+from .base import (
+    ParameterDefinition,
+    ParameterType,
+    ValidatorCategory,
+    ValidatorDefinition,
+)
+
+PROFILING_VALIDATORS: list[ValidatorDefinition] = [
+    ValidatorDefinition(
+        name="Cardinality",
+        display_name="Cardinality Validation",
+        category=ValidatorCategory.PROFILING,
+        description="Validates that column cardinality (distinct count) falls within expected bounds.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column to check cardinality",
+            ),
+            ParameterDefinition(
+                name="min_cardinality",
+                label="Minimum Cardinality",
+                type=ParameterType.INTEGER,
+                min_value=0,
+                description="Minimum expected distinct values",
+            ),
+            ParameterDefinition(
+                name="max_cardinality",
+                label="Maximum Cardinality",
+                type=ParameterType.INTEGER,
+                min_value=0,
+                description="Maximum expected distinct values",
+            ),
+        ],
+        tags=["profiling", "cardinality", "distinct", "count"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="UniquenessRatio",
+        display_name="Uniqueness Ratio",
+        category=ValidatorCategory.PROFILING,
+        description="Validates that the ratio of unique values to total values meets expectations.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column to analyze",
+            ),
+            ParameterDefinition(
+                name="min_ratio",
+                label="Minimum Ratio",
+                type=ParameterType.FLOAT,
+                min_value=0,
+                max_value=1,
+                description="Minimum uniqueness ratio (0-1)",
+            ),
+            ParameterDefinition(
+                name="max_ratio",
+                label="Maximum Ratio",
+                type=ParameterType.FLOAT,
+                min_value=0,
+                max_value=1,
+                description="Maximum uniqueness ratio (0-1)",
+            ),
+        ],
+        tags=["profiling", "uniqueness", "ratio", "distinct"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="Entropy",
+        display_name="Shannon Entropy",
+        category=ValidatorCategory.PROFILING,
+        description="Validates that column Shannon entropy falls within expected bounds.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column to calculate entropy",
+            ),
+            ParameterDefinition(
+                name="min_entropy",
+                label="Minimum Entropy",
+                type=ParameterType.FLOAT,
+                min_value=0,
+                description="Minimum expected entropy (bits)",
+            ),
+            ParameterDefinition(
+                name="max_entropy",
+                label="Maximum Entropy",
+                type=ParameterType.FLOAT,
+                min_value=0,
+                description="Maximum expected entropy (bits)",
+            ),
+        ],
+        tags=["profiling", "entropy", "information", "randomness"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="InformationGain",
+        display_name="Information Gain",
+        category=ValidatorCategory.PROFILING,
+        description="Validates information gain between two columns (for feature selection).",
+        parameters=[
+            ParameterDefinition(
+                name="feature_column",
+                label="Feature Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Feature column",
+            ),
+            ParameterDefinition(
+                name="target_column",
+                label="Target Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Target column",
+            ),
+            ParameterDefinition(
+                name="min_gain",
+                label="Minimum Gain",
+                type=ParameterType.FLOAT,
+                min_value=0,
+                description="Minimum required information gain",
+            ),
+        ],
+        tags=["profiling", "information_gain", "feature", "ml"],
+        severity_default="low",
+    ),
+    ValidatorDefinition(
+        name="ValueFrequency",
+        display_name="Value Frequency Distribution",
+        category=ValidatorCategory.PROFILING,
+        description="Validates that value frequency distribution meets expectations.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column to analyze",
+            ),
+            ParameterDefinition(
+                name="expected_distribution",
+                label="Expected Distribution",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "uniform", "label": "Uniform"},
+                    {"value": "normal", "label": "Normal (Gaussian)"},
+                    {"value": "exponential", "label": "Exponential"},
+                    {"value": "power_law", "label": "Power Law (Zipf)"},
+                ],
+                default="uniform",
+                description="Expected frequency distribution type",
+            ),
+            ParameterDefinition(
+                name="tolerance",
+                label="Tolerance",
+                type=ParameterType.FLOAT,
+                default=0.1,
+                min_value=0,
+                max_value=1,
+                description="Acceptable deviation from expected distribution",
+            ),
+        ],
+        tags=["profiling", "frequency", "distribution", "histogram"],
+        severity_default="low",
+    ),
+    ValidatorDefinition(
+        name="DistributionShape",
+        display_name="Distribution Shape",
+        category=ValidatorCategory.PROFILING,
+        description="Validates the shape of value distribution (skewness, kurtosis).",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column to analyze",
+            ),
+            ParameterDefinition(
+                name="max_skewness",
+                label="Maximum Skewness",
+                type=ParameterType.FLOAT,
+                description="Maximum acceptable skewness (absolute value)",
+            ),
+            ParameterDefinition(
+                name="max_kurtosis",
+                label="Maximum Kurtosis",
+                type=ParameterType.FLOAT,
+                description="Maximum acceptable excess kurtosis",
+            ),
+        ],
+        tags=["profiling", "distribution", "skewness", "kurtosis"],
+        severity_default="low",
+    ),
+    ValidatorDefinition(
+        name="MostCommonValue",
+        display_name="Most Common Value Check",
+        category=ValidatorCategory.PROFILING,
+        description="Validates the most common value and its frequency.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column to analyze",
+            ),
+            ParameterDefinition(
+                name="expected_value",
+                label="Expected Most Common Value",
+                type=ParameterType.STRING,
+                description="Expected most common value (optional)",
+            ),
+            ParameterDefinition(
+                name="max_frequency_ratio",
+                label="Maximum Frequency Ratio",
+                type=ParameterType.FLOAT,
+                min_value=0,
+                max_value=1,
+                description="Maximum ratio of most common value (detect imbalance)",
+            ),
+        ],
+        tags=["profiling", "mode", "common", "frequency"],
+        severity_default="low",
+    ),
+    ValidatorDefinition(
+        name="DataTypeConsistency",
+        display_name="Data Type Consistency",
+        category=ValidatorCategory.PROFILING,
+        description="Validates that inferred data types are consistent with expected types.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column to validate",
+            ),
+            ParameterDefinition(
+                name="expected_type",
+                label="Expected Data Type",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "integer", "label": "Integer"},
+                    {"value": "float", "label": "Float"},
+                    {"value": "string", "label": "String"},
+                    {"value": "boolean", "label": "Boolean"},
+                    {"value": "date", "label": "Date"},
+                    {"value": "datetime", "label": "DateTime"},
+                    {"value": "categorical", "label": "Categorical"},
+                ],
+                required=True,
+                description="Expected inferred data type",
+            ),
+        ],
+        tags=["profiling", "type", "consistency", "inference"],
+        severity_default="medium",
+    ),
+]

truthound_dashboard/schemas/validators/referential_validators.py

@@ -0,0 +1,312 @@
+"""Referential validators.
+
+Validators for foreign key relationships, orphan detection, and hierarchy integrity.
+
+Import path in truthound: `from truthound.validators.referential import *`
+"""
+
+from .base import (
+    ParameterDefinition,
+    ParameterType,
+    ValidatorCategory,
+    ValidatorDefinition,
+)
+
+REFERENTIAL_VALIDATORS: list[ValidatorDefinition] = [
+    ValidatorDefinition(
+        name="ForeignKey",
+        display_name="Foreign Key",
+        category=ValidatorCategory.REFERENTIAL,
+        description="Validates that all values in a column exist in a reference table's column.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Foreign Key Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing foreign key values",
+            ),
+            ParameterDefinition(
+                name="reference_source",
+                label="Reference Data Source",
+                type=ParameterType.SOURCE_REF,
+                required=True,
+                description="Reference data source containing the primary key",
+            ),
+            ParameterDefinition(
+                name="reference_column",
+                label="Reference Column",
+                type=ParameterType.STRING,
+                required=True,
+                description="Primary key column in the reference data",
+            ),
+        ],
+        tags=["referential", "foreign_key", "fk", "relationship"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="CompositeForeignKey",
+        display_name="Composite Foreign Key",
+        category=ValidatorCategory.REFERENTIAL,
+        description="Validates composite foreign key relationships across multiple columns.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Foreign Key Columns",
+                type=ParameterType.COLUMN_LIST,
+                required=True,
+                description="Columns forming the composite foreign key",
+            ),
+            ParameterDefinition(
+                name="reference_source",
+                label="Reference Data Source",
+                type=ParameterType.SOURCE_REF,
+                required=True,
+                description="Reference data source containing the composite primary key",
+            ),
+            ParameterDefinition(
+                name="reference_columns",
+                label="Reference Columns",
+                type=ParameterType.STRING_LIST,
+                required=True,
+                description="Columns forming the composite primary key in reference",
+            ),
+        ],
+        tags=["referential", "composite_key", "foreign_key", "relationship"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="SelfReferentialFK",
+        display_name="Self-Referential Foreign Key",
+        category=ValidatorCategory.REFERENTIAL,
+        description="Validates self-referential foreign keys (e.g., parent_id references id in same table).",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Foreign Key Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing self-referential foreign key (e.g., parent_id)",
+            ),
+            ParameterDefinition(
+                name="reference_column",
+                label="Primary Key Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Primary key column being referenced (e.g., id)",
+            ),
+            ParameterDefinition(
+                name="allow_null",
+                label="Allow Null Values",
+                type=ParameterType.BOOLEAN,
+                default=True,
+                description="Whether null values are allowed (root nodes)",
+            ),
+        ],
+        tags=["referential", "self_referential", "hierarchy", "tree"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="OrphanRecord",
+        display_name="Orphan Record Detection",
+        category=ValidatorCategory.REFERENTIAL,
+        description="Detects orphan records that reference non-existent parent records.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Foreign Key Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing foreign key values",
+            ),
+            ParameterDefinition(
+                name="reference_source",
+                label="Reference Data Source",
+                type=ParameterType.SOURCE_REF,
+                required=True,
+                description="Parent data source",
+            ),
+            ParameterDefinition(
+                name="reference_column",
+                label="Reference Column",
+                type=ParameterType.STRING,
+                required=True,
+                description="Primary key column in the parent data",
+            ),
+        ],
+        tags=["referential", "orphan", "integrity", "dangling"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="DanglingReference",
+        display_name="Dangling Reference Detection",
+        category=ValidatorCategory.REFERENTIAL,
+        description="Detects references pointing to deleted or non-existent records.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Reference Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing reference values",
+            ),
+            ParameterDefinition(
+                name="reference_source",
+                label="Reference Data Source",
+                type=ParameterType.SOURCE_REF,
+                required=True,
+                description="Target data source",
+            ),
+            ParameterDefinition(
+                name="reference_column",
+                label="Target Column",
+                type=ParameterType.STRING,
+                required=True,
+                description="Column in target data being referenced",
+            ),
+        ],
+        tags=["referential", "dangling", "broken_link", "integrity"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="CircularReference",
+        display_name="Circular Reference Detection",
+        category=ValidatorCategory.REFERENTIAL,
+        description="Detects circular references in self-referential data (e.g., A→B→C→A).",
+        parameters=[
+            ParameterDefinition(
+                name="id_column",
+                label="ID Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Primary key column (e.g., id)",
+            ),
+            ParameterDefinition(
+                name="parent_column",
+                label="Parent Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column referencing parent (e.g., parent_id)",
+            ),
+            ParameterDefinition(
+                name="max_depth",
+                label="Maximum Detection Depth",
+                type=ParameterType.INTEGER,
+                default=100,
+                min_value=1,
+                max_value=1000,
+                description="Maximum depth to search for cycles",
+            ),
+        ],
+        tags=["referential", "circular", "cycle", "hierarchy"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="HierarchyDepth",
+        display_name="Hierarchy Depth Validation",
+        category=ValidatorCategory.REFERENTIAL,
+        description="Validates that hierarchy depth doesn't exceed a maximum limit.",
+        parameters=[
+            ParameterDefinition(
+                name="id_column",
+                label="ID Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Primary key column",
+            ),
+            ParameterDefinition(
+                name="parent_column",
+                label="Parent Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column referencing parent",
+            ),
+            ParameterDefinition(
+                name="max_depth",
+                label="Maximum Allowed Depth",
+                type=ParameterType.INTEGER,
+                required=True,
+                min_value=1,
+                max_value=100,
+                description="Maximum allowed hierarchy depth",
+            ),
+        ],
+        tags=["referential", "hierarchy", "depth", "tree"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="CascadeIntegrity",
+        display_name="Cascade Integrity",
+        category=ValidatorCategory.REFERENTIAL,
+        description="Validates cascade delete/update integrity across related tables.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Foreign Key Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column with cascade relationship",
+            ),
+            ParameterDefinition(
+                name="reference_source",
+                label="Reference Data Source",
+                type=ParameterType.SOURCE_REF,
+                required=True,
+                description="Parent data source",
+            ),
+            ParameterDefinition(
+                name="reference_column",
+                label="Reference Column",
+                type=ParameterType.STRING,
+                required=True,
+                description="Primary key in parent",
+            ),
+            ParameterDefinition(
+                name="action",
+                label="Cascade Action",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "cascade", "label": "Cascade"},
+                    {"value": "set_null", "label": "Set Null"},
+                    {"value": "restrict", "label": "Restrict"},
+                    {"value": "no_action", "label": "No Action"},
+                ],
+                default="cascade",
+                description="Expected cascade behavior",
+            ),
+        ],
+        tags=["referential", "cascade", "delete", "update"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="MultiTableOrphan",
+        display_name="Multi-Table Orphan Check",
+        category=ValidatorCategory.REFERENTIAL,
+        description="Detects orphan records across multiple related tables.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Foreign Key Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing foreign key",
+            ),
+            ParameterDefinition(
+                name="reference_sources",
+                label="Reference Data Sources",
+                type=ParameterType.STRING_LIST,
+                required=True,
+                description="List of parent data source IDs",
+            ),
+            ParameterDefinition(
+                name="reference_columns",
+                label="Reference Columns",
+                type=ParameterType.STRING_LIST,
+                required=True,
+                description="Corresponding primary key columns",
+            ),
+        ],
+        tags=["referential", "orphan", "multi_table", "integrity"],
+        severity_default="high",
+    ),
+]