truthound-dashboard 1.2.1__py3-none-any.whl → 1.3.1__py3-none-any.whl

truthound_dashboard/schemas/validators/privacy_validators.py

@@ -0,0 +1,531 @@
+"""Privacy validators.
+
+Validators for data privacy compliance (GDPR, CCPA, LGPD) and PII detection.
+"""
+
+from .base import (
+    ParameterDefinition,
+    ParameterType,
+    ValidatorCategory,
+    ValidatorDefinition,
+)
+
+PRIVACY_VALIDATORS: list[ValidatorDefinition] = [
+    # PII Detection
+    ValidatorDefinition(
+        name="PIIDetection",
+        display_name="PII Detection",
+        category=ValidatorCategory.PRIVACY,
+        description="Detects Personally Identifiable Information (PII) in columns.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns to Scan",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns to scan (leave empty for all string columns)",
+            ),
+            ParameterDefinition(
+                name="pii_types",
+                label="PII Types",
+                type=ParameterType.MULTI_SELECT,
+                description="Types of PII to detect",
+                options=[
+                    {"value": "email", "label": "Email Address"},
+                    {"value": "phone", "label": "Phone Number"},
+                    {"value": "ssn", "label": "Social Security Number"},
+                    {"value": "credit_card", "label": "Credit Card Number"},
+                    {"value": "ip_address", "label": "IP Address"},
+                    {"value": "name", "label": "Personal Name"},
+                    {"value": "address", "label": "Physical Address"},
+                    {"value": "date_of_birth", "label": "Date of Birth"},
+                    {"value": "passport", "label": "Passport Number"},
+                    {"value": "driver_license", "label": "Driver's License"},
+                ],
+            ),
+            ParameterDefinition(
+                name="sample_size",
+                label="Sample Size",
+                type=ParameterType.INTEGER,
+                description="Number of rows to sample (0 for all)",
+                default=1000,
+                min_value=0,
+            ),
+            ParameterDefinition(
+                name="fail_on_detection",
+                label="Fail on Detection",
+                type=ParameterType.BOOLEAN,
+                description="Fail validation if PII is detected",
+                default=True,
+            ),
+        ],
+        tags=["privacy", "pii", "detection", "scan"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="EmailPII",
+        display_name="Email PII",
+        category=ValidatorCategory.PRIVACY,
+        description="Detects unprotected email addresses.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns to check (leave empty for all)",
+            ),
+            ParameterDefinition(
+                name="allow_hashed",
+                label="Allow Hashed",
+                type=ParameterType.BOOLEAN,
+                description="Allow hashed/masked email formats",
+                default=True,
+            ),
+            ParameterDefinition(
+                name="allowed_domains",
+                label="Allowed Domains",
+                type=ParameterType.STRING_LIST,
+                description="Domains to whitelist (e.g., company.com)",
+            ),
+        ],
+        tags=["privacy", "pii", "email"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="PhonePII",
+        display_name="Phone PII",
+        category=ValidatorCategory.PRIVACY,
+        description="Detects unprotected phone numbers.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+            ),
+            ParameterDefinition(
+                name="country_codes",
+                label="Country Codes",
+                type=ParameterType.STRING_LIST,
+                description="Country codes to detect (leave empty for all)",
+            ),
+            ParameterDefinition(
+                name="allow_masked",
+                label="Allow Masked",
+                type=ParameterType.BOOLEAN,
+                description="Allow masked formats (XXX-XXX-1234)",
+                default=True,
+            ),
+        ],
+        tags=["privacy", "pii", "phone"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="SSNDetection",
+        display_name="SSN Detection",
+        category=ValidatorCategory.PRIVACY,
+        description="Detects Social Security Numbers.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+            ),
+            ParameterDefinition(
+                name="allow_masked",
+                label="Allow Masked",
+                type=ParameterType.BOOLEAN,
+                description="Allow masked formats (XXX-XX-1234)",
+                default=True,
+            ),
+        ],
+        tags=["privacy", "pii", "ssn", "us"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="CreditCardDetection",
+        display_name="Credit Card Detection",
+        category=ValidatorCategory.PRIVACY,
+        description="Detects credit card numbers using Luhn algorithm.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+            ),
+            ParameterDefinition(
+                name="allow_masked",
+                label="Allow Masked",
+                type=ParameterType.BOOLEAN,
+                description="Allow last-4 digit formats",
+                default=True,
+            ),
+            ParameterDefinition(
+                name="validate_luhn",
+                label="Validate Luhn",
+                type=ParameterType.BOOLEAN,
+                description="Use Luhn algorithm validation",
+                default=True,
+            ),
+        ],
+        tags=["privacy", "pii", "credit_card", "pci"],
+        severity_default="critical",
+    ),
+    # Compliance Validators
+    ValidatorDefinition(
+        name="GDPRCompliance",
+        display_name="GDPR Compliance",
+        category=ValidatorCategory.PRIVACY,
+        description="Validates GDPR compliance requirements.",
+        parameters=[
+            ParameterDefinition(
+                name="check_pii",
+                label="Check PII",
+                type=ParameterType.BOOLEAN,
+                description="Scan for unprotected PII",
+                default=True,
+            ),
+            ParameterDefinition(
+                name="check_consent",
+                label="Check Consent Column",
+                type=ParameterType.COLUMN,
+                description="Column containing consent flag",
+            ),
+            ParameterDefinition(
+                name="check_retention",
+                label="Check Data Retention",
+                type=ParameterType.BOOLEAN,
+                description="Validate data retention policies",
+                default=False,
+            ),
+            ParameterDefinition(
+                name="retention_date_column",
+                label="Retention Date Column",
+                type=ParameterType.COLUMN,
+                description="Column with data collection/update date",
+                depends_on="check_retention",
+                depends_value=True,
+            ),
+            ParameterDefinition(
+                name="max_retention_days",
+                label="Max Retention Days",
+                type=ParameterType.INTEGER,
+                description="Maximum data retention period in days",
+                default=730,
+                min_value=1,
+                depends_on="check_retention",
+                depends_value=True,
+            ),
+            ParameterDefinition(
+                name="sensitive_columns",
+                label="Sensitive Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns containing sensitive data (Art. 9)",
+            ),
+        ],
+        tags=["privacy", "gdpr", "compliance", "eu"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="CCPACompliance",
+        display_name="CCPA Compliance",
+        category=ValidatorCategory.PRIVACY,
+        description="Validates California Consumer Privacy Act compliance.",
+        parameters=[
+            ParameterDefinition(
+                name="check_pii",
+                label="Check PII",
+                type=ParameterType.BOOLEAN,
+                default=True,
+            ),
+            ParameterDefinition(
+                name="check_opt_out",
+                label="Check Opt-Out Column",
+                type=ParameterType.COLUMN,
+                description="Column containing opt-out flag",
+            ),
+            ParameterDefinition(
+                name="check_sale_flag",
+                label="Check Sale Flag",
+                type=ParameterType.COLUMN,
+                description="Column indicating data sale consent",
+            ),
+            ParameterDefinition(
+                name="minor_age_column",
+                label="Minor Age Column",
+                type=ParameterType.COLUMN,
+                description="Column with age for minor protection",
+            ),
+        ],
+        tags=["privacy", "ccpa", "compliance", "california"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="LGPDCompliance",
+        display_name="LGPD Compliance",
+        category=ValidatorCategory.PRIVACY,
+        description="Validates Brazilian LGPD compliance requirements.",
+        parameters=[
+            ParameterDefinition(
+                name="check_pii",
+                label="Check PII",
+                type=ParameterType.BOOLEAN,
+                default=True,
+            ),
+            ParameterDefinition(
+                name="check_cpf",
+                label="Check CPF",
+                type=ParameterType.BOOLEAN,
+                description="Detect Brazilian CPF numbers",
+                default=True,
+            ),
+            ParameterDefinition(
+                name="check_consent",
+                label="Check Consent Column",
+                type=ParameterType.COLUMN,
+            ),
+            ParameterDefinition(
+                name="purpose_column",
+                label="Purpose Column",
+                type=ParameterType.COLUMN,
+                description="Column documenting data processing purpose",
+            ),
+        ],
+        tags=["privacy", "lgpd", "compliance", "brazil"],
+        severity_default="critical",
+    ),
+    # Data Retention
+    ValidatorDefinition(
+        name="DataRetention",
+        display_name="Data Retention",
+        category=ValidatorCategory.PRIVACY,
+        description="Validates data retention policies.",
+        parameters=[
+            ParameterDefinition(
+                name="date_column",
+                label="Date Column",
+                type=ParameterType.COLUMN,
+                description="Column with record creation/update date",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="max_age_days",
+                label="Max Age (Days)",
+                type=ParameterType.INTEGER,
+                description="Maximum allowed age in days",
+                required=True,
+                min_value=1,
+            ),
+            ParameterDefinition(
+                name="category_column",
+                label="Category Column",
+                type=ParameterType.COLUMN,
+                description="Column for category-based retention",
+            ),
+            ParameterDefinition(
+                name="category_retention",
+                label="Category Retention Rules",
+                type=ParameterType.SCHEMA,
+                description="JSON: {category: max_days}",
+                placeholder='{"financial": 2555, "marketing": 365}',
+                depends_on="category_column",
+            ),
+        ],
+        tags=["privacy", "retention", "policy"],
+        severity_default="high",
+    ),
+    # Masking & Anonymization
+    ValidatorDefinition(
+        name="MaskingVerification",
+        display_name="Masking Verification",
+        category=ValidatorCategory.PRIVACY,
+        description="Verifies that sensitive columns are properly masked.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="masking_type",
+                label="Masking Type",
+                type=ParameterType.SELECT,
+                required=True,
+                options=[
+                    {"value": "full", "label": "Full Masking (******)"},
+                    {"value": "partial", "label": "Partial Masking (***1234)"},
+                    {"value": "hash", "label": "Hash (SHA256, MD5)"},
+                    {"value": "tokenization", "label": "Tokenization (UUID)"},
+                    {"value": "redacted", "label": "Redacted ([REDACTED])"},
+                ],
+            ),
+            ParameterDefinition(
+                name="visible_chars",
+                label="Visible Characters",
+                type=ParameterType.INTEGER,
+                description="Number of visible characters (for partial masking)",
+                default=4,
+                min_value=0,
+                depends_on="masking_type",
+                depends_value="partial",
+            ),
+        ],
+        tags=["privacy", "masking", "anonymization"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="AnonymizationCheck",
+        display_name="Anonymization Check",
+        category=ValidatorCategory.PRIVACY,
+        description="Verifies k-anonymity and l-diversity requirements.",
+        parameters=[
+            ParameterDefinition(
+                name="quasi_identifiers",
+                label="Quasi-Identifiers",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns that could identify individuals indirectly",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="k_anonymity",
+                label="K-Anonymity",
+                type=ParameterType.INTEGER,
+                description="Minimum group size for k-anonymity",
+                default=5,
+                min_value=2,
+            ),
+            ParameterDefinition(
+                name="sensitive_column",
+                label="Sensitive Column",
+                type=ParameterType.COLUMN,
+                description="Column for l-diversity check",
+            ),
+            ParameterDefinition(
+                name="l_diversity",
+                label="L-Diversity",
+                type=ParameterType.INTEGER,
+                description="Minimum distinct values for l-diversity",
+                default=3,
+                min_value=2,
+                depends_on="sensitive_column",
+            ),
+        ],
+        tags=["privacy", "anonymization", "k_anonymity", "l_diversity"],
+        severity_default="high",
+    ),
+    # Sensitive Data Classification
+    ValidatorDefinition(
+        name="SensitiveDataClassification",
+        display_name="Sensitive Data Classification",
+        category=ValidatorCategory.PRIVACY,
+        description="Classifies and validates sensitive data handling.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns to Classify",
+                type=ParameterType.COLUMN_LIST,
+            ),
+            ParameterDefinition(
+                name="classification_levels",
+                label="Classification Levels",
+                type=ParameterType.MULTI_SELECT,
+                description="Expected classification levels",
+                options=[
+                    {"value": "public", "label": "Public"},
+                    {"value": "internal", "label": "Internal"},
+                    {"value": "confidential", "label": "Confidential"},
+                    {"value": "restricted", "label": "Restricted"},
+                    {"value": "pii", "label": "PII"},
+                    {"value": "phi", "label": "PHI (Health)"},
+                    {"value": "pci", "label": "PCI (Payment)"},
+                ],
+            ),
+            ParameterDefinition(
+                name="max_allowed_level",
+                label="Max Allowed Level",
+                type=ParameterType.SELECT,
+                description="Maximum classification level allowed in dataset",
+                options=[
+                    {"value": "public", "label": "Public Only"},
+                    {"value": "internal", "label": "Up to Internal"},
+                    {"value": "confidential", "label": "Up to Confidential"},
+                ],
+            ),
+        ],
+        tags=["privacy", "classification", "sensitive"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="AgeVerification",
+        display_name="Age Verification",
+        category=ValidatorCategory.PRIVACY,
+        description="Validates age-related privacy requirements (COPPA, etc.).",
+        parameters=[
+            ParameterDefinition(
+                name="age_column",
+                label="Age Column",
+                type=ParameterType.COLUMN,
+                description="Column containing age or birth date",
+                required=True,
+            ),
+            ParameterDefinition(
+                name="min_age",
+                label="Minimum Age",
+                type=ParameterType.INTEGER,
+                description="Minimum required age",
+                default=13,
+                min_value=0,
+            ),
+            ParameterDefinition(
+                name="consent_column",
+                label="Parental Consent Column",
+                type=ParameterType.COLUMN,
+                description="Column containing parental consent for minors",
+            ),
+            ParameterDefinition(
+                name="minor_threshold",
+                label="Minor Threshold Age",
+                type=ParameterType.INTEGER,
+                description="Age threshold for minor classification",
+                default=18,
+                min_value=0,
+            ),
+        ],
+        tags=["privacy", "age", "coppa", "minor"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="CrossBorderDataTransfer",
+        display_name="Cross-Border Data Transfer",
+        category=ValidatorCategory.PRIVACY,
+        description="Validates cross-border data transfer compliance.",
+        parameters=[
+            ParameterDefinition(
+                name="country_column",
+                label="Country Column",
+                type=ParameterType.COLUMN,
+                description="Column containing destination country",
+            ),
+            ParameterDefinition(
+                name="allowed_countries",
+                label="Allowed Countries",
+                type=ParameterType.STRING_LIST,
+                description="List of allowed destination countries",
+            ),
+            ParameterDefinition(
+                name="restricted_countries",
+                label="Restricted Countries",
+                type=ParameterType.STRING_LIST,
+                description="List of restricted countries (embargoed, etc.)",
+            ),
+            ParameterDefinition(
+                name="adequacy_countries",
+                label="Adequacy Countries",
+                type=ParameterType.STRING_LIST,
+                description="Countries with GDPR adequacy decisions",
+                default=["CA", "JP", "KR", "GB", "CH", "NZ", "IL", "AR", "UY"],
+            ),
+        ],
+        tags=["privacy", "cross_border", "transfer", "international"],
+        severity_default="high",
+    ),
+]