truthound-dashboard 1.2.1__py3-none-any.whl → 1.3.1__py3-none-any.whl

truthound_dashboard/schemas/validators/schema_validators.py

@@ -0,0 +1,408 @@
+"""Schema validators.
+
+Validators for structural integrity: column existence, types, ordering, relationships.
+"""
+
+from .base import (
+    ParameterDefinition,
+    ParameterType,
+    ValidatorCategory,
+    ValidatorDefinition,
+)
+
+SCHEMA_VALIDATORS: list[ValidatorDefinition] = [
+    ValidatorDefinition(
+        name="ColumnExists",
+        display_name="Column Exists",
+        category=ValidatorCategory.SCHEMA,
+        description="Validates that specified columns exist in the dataset.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Required Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Column names that must exist in the dataset",
+                required=True,
+            ),
+        ],
+        tags=["schema", "structure", "column"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="ColumnNotExists",
+        display_name="Column Not Exists",
+        category=ValidatorCategory.SCHEMA,
+        description="Ensures specified columns are absent (e.g., deprecated or sensitive fields).",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Forbidden Columns",
+                type=ParameterType.STRING_LIST,
+                description="Column names that must NOT exist in the dataset",
+                required=True,
+                placeholder="e.g., password, ssn, credit_card",
+            ),
+        ],
+        tags=["schema", "structure", "column", "security"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="ColumnCount",
+        display_name="Column Count",
+        category=ValidatorCategory.SCHEMA,
+        description="Validates the number of columns in the dataset.",
+        parameters=[
+            ParameterDefinition(
+                name="expected",
+                label="Expected Count",
+                type=ParameterType.INTEGER,
+                description="Exact expected column count (use min/max for range)",
+                min_value=0,
+            ),
+            ParameterDefinition(
+                name="min_count",
+                label="Minimum Count",
+                type=ParameterType.INTEGER,
+                description="Minimum acceptable column count",
+                min_value=0,
+            ),
+            ParameterDefinition(
+                name="max_count",
+                label="Maximum Count",
+                type=ParameterType.INTEGER,
+                description="Maximum acceptable column count",
+                min_value=0,
+            ),
+        ],
+        tags=["schema", "structure", "count"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="RowCount",
+        display_name="Row Count",
+        category=ValidatorCategory.SCHEMA,
+        description="Validates the number of rows in the dataset.",
+        parameters=[
+            ParameterDefinition(
+                name="expected",
+                label="Expected Count",
+                type=ParameterType.INTEGER,
+                description="Exact expected row count",
+                min_value=0,
+            ),
+            ParameterDefinition(
+                name="min_count",
+                label="Minimum Count",
+                type=ParameterType.INTEGER,
+                description="Minimum acceptable row count",
+                min_value=0,
+            ),
+            ParameterDefinition(
+                name="max_count",
+                label="Maximum Count",
+                type=ParameterType.INTEGER,
+                description="Maximum acceptable row count",
+                min_value=0,
+            ),
+        ],
+        tags=["schema", "structure", "count", "row"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="ColumnType",
+        display_name="Column Type",
+        category=ValidatorCategory.SCHEMA,
+        description="Validates that a column conforms to an expected data type.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                description="Target column to validate",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="expected_type",
+                label="Expected Type",
+                type=ParameterType.SELECT,
+                description="Expected Polars data type",
+                required=True,
+                options=[
+                    {"value": "Int8", "label": "Int8"},
+                    {"value": "Int16", "label": "Int16"},
+                    {"value": "Int32", "label": "Int32"},
+                    {"value": "Int64", "label": "Int64"},
+                    {"value": "UInt8", "label": "UInt8"},
+                    {"value": "UInt16", "label": "UInt16"},
+                    {"value": "UInt32", "label": "UInt32"},
+                    {"value": "UInt64", "label": "UInt64"},
+                    {"value": "Float32", "label": "Float32"},
+                    {"value": "Float64", "label": "Float64"},
+                    {"value": "Boolean", "label": "Boolean"},
+                    {"value": "String", "label": "String (Utf8)"},
+                    {"value": "Date", "label": "Date"},
+                    {"value": "Datetime", "label": "Datetime"},
+                    {"value": "Duration", "label": "Duration"},
+                    {"value": "Time", "label": "Time"},
+                    {"value": "Categorical", "label": "Categorical"},
+                    {"value": "Object", "label": "Object"},
+                ],
+            ),
+        ],
+        tags=["schema", "type", "column"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="ColumnOrder",
+        display_name="Column Order",
+        category=ValidatorCategory.SCHEMA,
+        description="Ensures columns appear in the specified order.",
+        parameters=[
+            ParameterDefinition(
+                name="expected_order",
+                label="Expected Order",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns in their expected order",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="strict",
+                label="Strict Mode",
+                type=ParameterType.BOOLEAN,
+                description="If true, no additional columns are allowed",
+                default=False,
+            ),
+        ],
+        tags=["schema", "structure", "order"],
+        severity_default="low",
+    ),
+    ValidatorDefinition(
+        name="TableSchema",
+        display_name="Table Schema",
+        category=ValidatorCategory.SCHEMA,
+        description="Validates the complete schema against a reference specification.",
+        parameters=[
+            ParameterDefinition(
+                name="schema",
+                label="Schema Definition",
+                type=ParameterType.SCHEMA,
+                description="Column name to type mapping (JSON format)",
+                required=True,
+                placeholder='{"id": "Int64", "name": "String", "email": "String"}',
+            ),
+            ParameterDefinition(
+                name="strict",
+                label="Strict Mode",
+                type=ParameterType.BOOLEAN,
+                description="Reject extra columns not in schema",
+                default=False,
+            ),
+        ],
+        tags=["schema", "structure", "complete"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="ColumnPair",
+        display_name="Column Pair Relationship",
+        category=ValidatorCategory.SCHEMA,
+        description="Validates relationships between two columns.",
+        parameters=[
+            ParameterDefinition(
+                name="column_a",
+                label="First Column",
+                type=ParameterType.COLUMN,
+                description="First column in the relationship",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="column_b",
+                label="Second Column",
+                type=ParameterType.COLUMN,
+                description="Second column in the relationship",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="relationship",
+                label="Relationship Type",
+                type=ParameterType.SELECT,
+                description="Expected relationship between columns",
+                required=True,
+                options=[
+                    {"value": "equal", "label": "Equal"},
+                    {"value": "not_equal", "label": "Not Equal"},
+                    {"value": "greater", "label": "A > B"},
+                    {"value": "less", "label": "A < B"},
+                    {"value": "greater_equal", "label": "A >= B"},
+                    {"value": "less_equal", "label": "A <= B"},
+                ],
+            ),
+        ],
+        tags=["schema", "relationship", "comparison"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="MultiColumnUnique",
+        display_name="Multi-Column Unique",
+        category=ValidatorCategory.SCHEMA,
+        description="Ensures uniqueness across a combination of columns (composite key).",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Composite Key Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns that form the composite unique key",
+                required=True,
+            ),
+        ],
+        tags=["schema", "uniqueness", "composite", "key"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="ReferentialIntegrity",
+        display_name="Referential Integrity",
+        category=ValidatorCategory.SCHEMA,
+        description="Validates foreign key relationships between tables.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Foreign Key Column",
+                type=ParameterType.COLUMN,
+                description="Foreign key column in the current table",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_source_id",
+                label="Reference Source",
+                type=ParameterType.SOURCE_REF,
+                description="ID of the source containing the reference table",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_column",
+                label="Reference Column",
+                type=ParameterType.STRING,
+                description="Primary key column in the reference table",
+                required=True,
+            ),
+        ],
+        tags=["schema", "foreign_key", "relationship", "integrity"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="MultiColumnSum",
+        display_name="Multi-Column Sum",
+        category=ValidatorCategory.SCHEMA,
+        description="Validates that the sum of specified columns equals an expected value.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns to Sum",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns whose values should be summed",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="expected_sum",
+                label="Expected Sum",
+                type=ParameterType.FLOAT,
+                description="Expected sum value",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="tolerance",
+                label="Tolerance",
+                type=ParameterType.FLOAT,
+                description="Acceptable tolerance for floating point comparison",
+                default=0.0001,
+                min_value=0,
+            ),
+        ],
+        tags=["schema", "arithmetic", "sum", "calculation"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="MultiColumnCalculation",
+        display_name="Multi-Column Calculation",
+        category=ValidatorCategory.SCHEMA,
+        description="Validates arbitrary arithmetic relationships between columns.",
+        parameters=[
+            ParameterDefinition(
+                name="expression",
+                label="Expression",
+                type=ParameterType.EXPRESSION,
+                description="Mathematical expression (e.g., 'price * quantity == total')",
+                required=True,
+                placeholder="price * quantity == total",
+            ),
+            ParameterDefinition(
+                name="tolerance",
+                label="Tolerance",
+                type=ParameterType.FLOAT,
+                description="Acceptable tolerance for comparison",
+                default=0.0001,
+                min_value=0,
+            ),
+        ],
+        tags=["schema", "arithmetic", "expression", "calculation"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="ColumnPairInSet",
+        display_name="Column Pair In Set",
+        category=ValidatorCategory.SCHEMA,
+        description="Validates that column value pairs exist within a predefined set.",
+        parameters=[
+            ParameterDefinition(
+                name="column_a",
+                label="First Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="column_b",
+                label="Second Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="valid_pairs",
+                label="Valid Pairs",
+                type=ParameterType.SCHEMA,
+                description="JSON array of valid [a, b] pairs",
+                required=True,
+                placeholder='[["US", "USD"], ["UK", "GBP"], ["EU", "EUR"]]',
+            ),
+        ],
+        tags=["schema", "pair", "set", "validation"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="ColumnPairNotInSet",
+        display_name="Column Pair Not In Set",
+        category=ValidatorCategory.SCHEMA,
+        description="Ensures column value pairs do not exist within a forbidden set.",
+        parameters=[
+            ParameterDefinition(
+                name="column_a",
+                label="First Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="column_b",
+                label="Second Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="forbidden_pairs",
+                label="Forbidden Pairs",
+                type=ParameterType.SCHEMA,
+                description="JSON array of forbidden [a, b] pairs",
+                required=True,
+            ),
+        ],
+        tags=["schema", "pair", "set", "forbidden"],
+        severity_default="high",
+    ),
+]