truthound-dashboard 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

truthound_dashboard/schemas/validators/cross_table_validators.py

@@ -0,0 +1,375 @@
+"""Cross-table validators.
+
+Validators for multi-table relationships, foreign key constraints, and cross-dataset comparisons.
+These validators require reference to another data source.
+"""
+
+from .base import (
+    ParameterDefinition,
+    ParameterType,
+    ValidatorCategory,
+    ValidatorDefinition,
+)
+
+CROSS_TABLE_VALIDATORS: list[ValidatorDefinition] = [
+    ValidatorDefinition(
+        name="ForeignKey",
+        display_name="Foreign Key",
+        category=ValidatorCategory.CROSS_TABLE,
+        description="Validates that all values in a column exist in a reference table's column.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Foreign Key Column",
+                type=ParameterType.COLUMN,
+                description="Column containing foreign key values",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_source_id",
+                label="Reference Source",
+                type=ParameterType.SOURCE_REF,
+                description="ID of the data source containing the reference table",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_column",
+                label="Reference Column",
+                type=ParameterType.STRING,
+                description="Primary/unique key column in the reference table",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="allow_null",
+                label="Allow Null",
+                type=ParameterType.BOOLEAN,
+                description="Allow null values in foreign key column",
+                default=True,
+            ),
+        ],
+        tags=["cross_table", "foreign_key", "referential", "integrity"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="CompositeForeignKey",
+        display_name="Composite Foreign Key",
+        category=ValidatorCategory.CROSS_TABLE,
+        description="Validates composite foreign key relationships across multiple columns.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Foreign Key Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns forming the composite foreign key",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_source_id",
+                label="Reference Source",
+                type=ParameterType.SOURCE_REF,
+                description="ID of the data source containing the reference table",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_columns",
+                label="Reference Columns",
+                type=ParameterType.STRING_LIST,
+                description="Columns in the reference table (same order as foreign key columns)",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="allow_null",
+                label="Allow Null",
+                type=ParameterType.BOOLEAN,
+                description="Allow null values in any foreign key column",
+                default=True,
+            ),
+        ],
+        tags=["cross_table", "foreign_key", "composite", "referential"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="Orphan",
+        display_name="Orphan Records",
+        category=ValidatorCategory.CROSS_TABLE,
+        description="Detects orphan records that have no matching parent record.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Child Column",
+                type=ParameterType.COLUMN,
+                description="Column in child table referencing parent",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_source_id",
+                label="Parent Source",
+                type=ParameterType.SOURCE_REF,
+                description="ID of the parent data source",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_column",
+                label="Parent Column",
+                type=ParameterType.STRING,
+                description="Primary key column in parent table",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="ignore_null",
+                label="Ignore Null Values",
+                type=ParameterType.BOOLEAN,
+                description="Do not flag null values as orphans",
+                default=True,
+            ),
+        ],
+        tags=["cross_table", "orphan", "referential", "integrity"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="CrossTableRowCount",
+        display_name="Cross-Table Row Count",
+        category=ValidatorCategory.CROSS_TABLE,
+        description="Compares row counts between two tables with optional operators.",
+        parameters=[
+            ParameterDefinition(
+                name="reference_source_id",
+                label="Reference Source",
+                type=ParameterType.SOURCE_REF,
+                description="ID of the reference data source",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="operator",
+                label="Comparison Operator",
+                type=ParameterType.SELECT,
+                description="How to compare row counts",
+                required=True,
+                options=[
+                    {"value": "eq", "label": "Equal (=)"},
+                    {"value": "ne", "label": "Not Equal (!=)"},
+                    {"value": "gt", "label": "Greater Than (>)"},
+                    {"value": "gte", "label": "Greater Than or Equal (>=)"},
+                    {"value": "lt", "label": "Less Than (<)"},
+                    {"value": "lte", "label": "Less Than or Equal (<=)"},
+                ],
+                default="eq",
+            ),
+            ParameterDefinition(
+                name="tolerance",
+                label="Tolerance",
+                type=ParameterType.FLOAT,
+                description="Acceptable tolerance as ratio (e.g., 0.05 for 5%)",
+                default=0,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["cross_table", "row_count", "comparison"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="CrossTableAggregate",
+        display_name="Cross-Table Aggregate",
+        category=ValidatorCategory.CROSS_TABLE,
+        description="Compares aggregate statistics between corresponding columns in two tables.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                description="Column in current table to aggregate",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_source_id",
+                label="Reference Source",
+                type=ParameterType.SOURCE_REF,
+                description="ID of the reference data source",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_column",
+                label="Reference Column",
+                type=ParameterType.STRING,
+                description="Column in reference table (default: same name)",
+            ),
+            ParameterDefinition(
+                name="aggregate",
+                label="Aggregate Function",
+                type=ParameterType.SELECT,
+                description="Aggregate function to compare",
+                required=True,
+                options=[
+                    {"value": "sum", "label": "Sum"},
+                    {"value": "mean", "label": "Mean"},
+                    {"value": "median", "label": "Median"},
+                    {"value": "min", "label": "Minimum"},
+                    {"value": "max", "label": "Maximum"},
+                    {"value": "count", "label": "Count"},
+                    {"value": "std", "label": "Standard Deviation"},
+                ],
+            ),
+            ParameterDefinition(
+                name="operator",
+                label="Comparison Operator",
+                type=ParameterType.SELECT,
+                description="How to compare aggregates",
+                required=True,
+                options=[
+                    {"value": "eq", "label": "Equal (=)"},
+                    {"value": "ne", "label": "Not Equal (!=)"},
+                    {"value": "gt", "label": "Greater Than (>)"},
+                    {"value": "gte", "label": "Greater Than or Equal (>=)"},
+                    {"value": "lt", "label": "Less Than (<)"},
+                    {"value": "lte", "label": "Less Than or Equal (<=)"},
+                ],
+                default="eq",
+            ),
+            ParameterDefinition(
+                name="tolerance",
+                label="Tolerance",
+                type=ParameterType.FLOAT,
+                description="Acceptable tolerance as ratio",
+                default=0.01,
+                min_value=0,
+            ),
+        ],
+        tags=["cross_table", "aggregate", "comparison", "statistics"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="CrossTableDistinct",
+        display_name="Cross-Table Distinct Values",
+        category=ValidatorCategory.CROSS_TABLE,
+        description="Compares distinct value counts between two tables.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                description="Column to check distinct values",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_source_id",
+                label="Reference Source",
+                type=ParameterType.SOURCE_REF,
+                description="ID of the reference data source",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_column",
+                label="Reference Column",
+                type=ParameterType.STRING,
+                description="Column in reference table (default: same name)",
+            ),
+            ParameterDefinition(
+                name="operator",
+                label="Comparison Operator",
+                type=ParameterType.SELECT,
+                description="How to compare distinct counts",
+                required=True,
+                options=[
+                    {"value": "eq", "label": "Equal (=)"},
+                    {"value": "subset", "label": "Subset Of"},
+                    {"value": "superset", "label": "Superset Of"},
+                    {"value": "intersect", "label": "Has Intersection"},
+                    {"value": "disjoint", "label": "Disjoint (No Overlap)"},
+                ],
+                default="eq",
+            ),
+        ],
+        tags=["cross_table", "distinct", "comparison"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="CrossTableSchema",
+        display_name="Cross-Table Schema Match",
+        category=ValidatorCategory.CROSS_TABLE,
+        description="Validates that two tables have compatible schemas.",
+        parameters=[
+            ParameterDefinition(
+                name="reference_source_id",
+                label="Reference Source",
+                type=ParameterType.SOURCE_REF,
+                description="ID of the reference data source",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="check_columns",
+                label="Check Column Names",
+                type=ParameterType.BOOLEAN,
+                description="Verify column names match",
+                default=True,
+            ),
+            ParameterDefinition(
+                name="check_types",
+                label="Check Column Types",
+                type=ParameterType.BOOLEAN,
+                description="Verify column types match",
+                default=True,
+            ),
+            ParameterDefinition(
+                name="check_order",
+                label="Check Column Order",
+                type=ParameterType.BOOLEAN,
+                description="Verify columns are in same order",
+                default=False,
+            ),
+            ParameterDefinition(
+                name="strict",
+                label="Strict Mode",
+                type=ParameterType.BOOLEAN,
+                description="Require exact match (no extra columns)",
+                default=False,
+            ),
+        ],
+        tags=["cross_table", "schema", "compatibility"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="JoinIntegrity",
+        display_name="Join Integrity",
+        category=ValidatorCategory.CROSS_TABLE,
+        description="Validates join integrity (cardinality) between two tables.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Join Column",
+                type=ParameterType.COLUMN,
+                description="Column in current table for joining",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_source_id",
+                label="Reference Source",
+                type=ParameterType.SOURCE_REF,
+                description="ID of the reference data source",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="reference_column",
+                label="Reference Join Column",
+                type=ParameterType.STRING,
+                description="Column in reference table for joining",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="expected_cardinality",
+                label="Expected Cardinality",
+                type=ParameterType.SELECT,
+                description="Expected join relationship type",
+                required=True,
+                options=[
+                    {"value": "one_to_one", "label": "One-to-One"},
+                    {"value": "one_to_many", "label": "One-to-Many"},
+                    {"value": "many_to_one", "label": "Many-to-One"},
+                    {"value": "many_to_many", "label": "Many-to-Many"},
+                ],
+            ),
+        ],
+        tags=["cross_table", "join", "cardinality", "relationship"],
+        severity_default="high",
+    ),
+]

truthound_dashboard/schemas/validators/datetime_validators.py

@@ -0,0 +1,253 @@
+"""Datetime validators.
+
+Validators for date/time format, range, ordering, and freshness.
+"""
+
+from .base import (
+    ParameterDefinition,
+    ParameterType,
+    ValidatorCategory,
+    ValidatorDefinition,
+)
+
+DATETIME_VALIDATORS: list[ValidatorDefinition] = [
+    ValidatorDefinition(
+        name="DateFormat",
+        display_name="Date Format",
+        category=ValidatorCategory.DATETIME,
+        description="Validates date/datetime format.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="format",
+                label="Date Format",
+                type=ParameterType.STRING,
+                description="Expected strptime format",
+                required=True,
+                placeholder="%Y-%m-%d",
+            ),
+        ],
+        tags=["datetime", "format"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="DateBetween",
+        display_name="Date Between",
+        category=ValidatorCategory.DATETIME,
+        description="Validates dates within a specified range.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="min_date",
+                label="Minimum Date",
+                type=ParameterType.DATE,
+                description="Minimum date (YYYY-MM-DD)",
+                placeholder="2020-01-01",
+            ),
+            ParameterDefinition(
+                name="max_date",
+                label="Maximum Date",
+                type=ParameterType.DATE,
+                description="Maximum date (YYYY-MM-DD)",
+                placeholder="2025-12-31",
+            ),
+        ],
+        tags=["datetime", "range", "bounds"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="FutureDate",
+        display_name="Future Date",
+        category=ValidatorCategory.DATETIME,
+        description="Ensures dates are in the future.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+        ],
+        tags=["datetime", "future"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="PastDate",
+        display_name="Past Date",
+        category=ValidatorCategory.DATETIME,
+        description="Ensures dates are in the past.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+        ],
+        tags=["datetime", "past"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="DateOrder",
+        display_name="Date Order",
+        category=ValidatorCategory.DATETIME,
+        description="Validates chronological ordering between date columns.",
+        parameters=[
+            ParameterDefinition(
+                name="start_column",
+                label="Start Date Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="end_column",
+                label="End Date Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+        ],
+        tags=["datetime", "order", "chronological"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="Timezone",
+        display_name="Timezone",
+        category=ValidatorCategory.DATETIME,
+        description="Validates timezone-aware datetime values.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="expected_timezone",
+                label="Expected Timezone",
+                type=ParameterType.STRING,
+                description="Expected timezone (e.g., 'UTC', 'America/New_York')",
+                placeholder="UTC",
+            ),
+        ],
+        tags=["datetime", "timezone"],
+        severity_default="low",
+    ),
+    ValidatorDefinition(
+        name="RecentData",
+        display_name="Recent Data",
+        category=ValidatorCategory.DATETIME,
+        description="Ensures data contains recent entries.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Datetime Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="max_age_days",
+                label="Maximum Age (Days)",
+                type=ParameterType.INTEGER,
+                description="Maximum age in days",
+                required=True,
+                min_value=1,
+            ),
+        ],
+        tags=["datetime", "recent", "freshness"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="DatePartCoverage",
+        display_name="Date Part Coverage",
+        category=ValidatorCategory.DATETIME,
+        description="Validates coverage across date parts.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Datetime Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="date_part",
+                label="Date Part",
+                type=ParameterType.SELECT,
+                required=True,
+                options=[
+                    {"value": "day", "label": "Day of Month"},
+                    {"value": "weekday", "label": "Day of Week"},
+                    {"value": "month", "label": "Month"},
+                    {"value": "hour", "label": "Hour"},
+                    {"value": "quarter", "label": "Quarter"},
+                ],
+            ),
+            ParameterDefinition(
+                name="min_coverage",
+                label="Minimum Coverage",
+                type=ParameterType.FLOAT,
+                description="Minimum coverage ratio (0.0-1.0)",
+                required=True,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["datetime", "coverage", "completeness"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="GroupedRecentData",
+        display_name="Grouped Recent Data",
+        category=ValidatorCategory.DATETIME,
+        description="Validates recency within groups.",
+        parameters=[
+            ParameterDefinition(
+                name="datetime_column",
+                label="Datetime Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="group_column",
+                label="Group Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="max_age_days",
+                label="Maximum Age (Days)",
+                type=ParameterType.INTEGER,
+                description="Maximum age per group in days",
+                required=True,
+                min_value=1,
+            ),
+        ],
+        tags=["datetime", "recent", "group"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="DatetimeParseable",
+        display_name="Datetime Parseable",
+        category=ValidatorCategory.DATETIME,
+        description="Validates that strings can be parsed as dates using dateutil.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+        ],
+        tags=["datetime", "parseable", "format"],
+        severity_default="medium",
+    ),
+]