truthound-dashboard 1.4.4__py3-none-any.whl → 1.5.0__py3-none-any.whl

truthound_dashboard/schemas/validators/localization_validators.py

@@ -0,0 +1,273 @@
+"""Localization validators.
+
+Validators for regional identifier formats including Korean, Japanese,
+and Chinese specific formats.
+
+Import path in truthound: `from truthound.validators.localization import *`
+"""
+
+from .base import (
+    ParameterDefinition,
+    ParameterType,
+    ValidatorCategory,
+    ValidatorDefinition,
+)
+
+LOCALIZATION_VALIDATORS: list[ValidatorDefinition] = [
+    # Korean validators
+    ValidatorDefinition(
+        name="KoreanBusinessNumber",
+        display_name="Korean Business Registration Number",
+        category=ValidatorCategory.LOCALIZATION,
+        description="Validates Korean business registration numbers (사업자등록번호, 10 digits).",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing business registration numbers",
+            ),
+            ParameterDefinition(
+                name="allow_hyphen",
+                label="Allow Hyphens",
+                type=ParameterType.BOOLEAN,
+                default=True,
+                description="Accept numbers with hyphens (123-45-67890)",
+            ),
+        ],
+        tags=["localization", "korean", "business", "registration"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="KoreanRRN",
+        display_name="Korean Resident Registration Number",
+        category=ValidatorCategory.LOCALIZATION,
+        description="Validates Korean Resident Registration Numbers (주민등록번호, 13 digits).",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing RRN values",
+            ),
+            ParameterDefinition(
+                name="masked",
+                label="Masked Format",
+                type=ParameterType.BOOLEAN,
+                default=True,
+                description="Accept partially masked numbers (123456-1******)",
+            ),
+        ],
+        tags=["localization", "korean", "rrn", "pii", "identity"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="KoreanPhone",
+        display_name="Korean Phone Number",
+        category=ValidatorCategory.LOCALIZATION,
+        description="Validates Korean phone number formats (mobile and landline).",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing phone numbers",
+            ),
+            ParameterDefinition(
+                name="type",
+                label="Phone Type",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "any", "label": "Any (Mobile or Landline)"},
+                    {"value": "mobile", "label": "Mobile (010/011/016/017/018/019)"},
+                    {"value": "landline", "label": "Landline (02/031-064)"},
+                ],
+                default="any",
+                description="Accepted phone number types",
+            ),
+        ],
+        tags=["localization", "korean", "phone", "contact"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="KoreanBankAccount",
+        display_name="Korean Bank Account Number",
+        category=ValidatorCategory.LOCALIZATION,
+        description="Validates Korean bank account number formats by bank.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing bank account numbers",
+            ),
+            ParameterDefinition(
+                name="bank",
+                label="Bank",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "any", "label": "Any Bank"},
+                    {"value": "kb", "label": "KB Kookmin Bank"},
+                    {"value": "shinhan", "label": "Shinhan Bank"},
+                    {"value": "woori", "label": "Woori Bank"},
+                    {"value": "hana", "label": "Hana Bank"},
+                    {"value": "nh", "label": "NH Bank"},
+                    {"value": "ibk", "label": "IBK"},
+                    {"value": "kakao", "label": "Kakao Bank"},
+                    {"value": "toss", "label": "Toss Bank"},
+                ],
+                default="any",
+                description="Bank for format validation",
+            ),
+        ],
+        tags=["localization", "korean", "bank", "financial"],
+        severity_default="high",
+    ),
+    # Japanese validators
+    ValidatorDefinition(
+        name="JapanesePostalCode",
+        display_name="Japanese Postal Code",
+        category=ValidatorCategory.LOCALIZATION,
+        description="Validates Japanese postal codes (〒xxx-xxxx format).",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing postal codes",
+            ),
+            ParameterDefinition(
+                name="allow_symbol",
+                label="Allow 〒 Symbol",
+                type=ParameterType.BOOLEAN,
+                default=True,
+                description="Accept codes with 〒 prefix",
+            ),
+        ],
+        tags=["localization", "japanese", "postal", "address"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="JapaneseMyNumber",
+        display_name="Japanese My Number",
+        category=ValidatorCategory.LOCALIZATION,
+        description="Validates Japanese Individual Number (マイナンバー, 12 digits).",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing My Number values",
+            ),
+        ],
+        tags=["localization", "japanese", "mynumber", "pii", "identity"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="JapanesePhone",
+        display_name="Japanese Phone Number",
+        category=ValidatorCategory.LOCALIZATION,
+        description="Validates Japanese phone number formats.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing phone numbers",
+            ),
+            ParameterDefinition(
+                name="type",
+                label="Phone Type",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "any", "label": "Any"},
+                    {"value": "mobile", "label": "Mobile (070/080/090)"},
+                    {"value": "landline", "label": "Landline"},
+                    {"value": "toll_free", "label": "Toll Free (0120/0800)"},
+                ],
+                default="any",
+                description="Accepted phone number types",
+            ),
+        ],
+        tags=["localization", "japanese", "phone", "contact"],
+        severity_default="medium",
+    ),
+    # Chinese validators
+    ValidatorDefinition(
+        name="ChineseID",
+        display_name="Chinese ID Number",
+        category=ValidatorCategory.LOCALIZATION,
+        description="Validates Chinese Resident Identity Card numbers (18 digits).",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing ID numbers",
+            ),
+            ParameterDefinition(
+                name="validate_region",
+                label="Validate Region Code",
+                type=ParameterType.BOOLEAN,
+                default=True,
+                description="Validate the 6-digit region code",
+            ),
+        ],
+        tags=["localization", "chinese", "id", "pii", "identity"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="ChineseUSCC",
+        display_name="Chinese Unified Social Credit Code",
+        category=ValidatorCategory.LOCALIZATION,
+        description="Validates Chinese USCC (统一社会信用代码, 18 characters).",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing USCC values",
+            ),
+        ],
+        tags=["localization", "chinese", "uscc", "business", "registration"],
+        severity_default="high",
+    ),
+    ValidatorDefinition(
+        name="ChinesePhone",
+        display_name="Chinese Phone Number",
+        category=ValidatorCategory.LOCALIZATION,
+        description="Validates Chinese phone number formats.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Column containing phone numbers",
+            ),
+            ParameterDefinition(
+                name="type",
+                label="Phone Type",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "any", "label": "Any"},
+                    {"value": "mobile", "label": "Mobile (1xx)"},
+                    {"value": "landline", "label": "Landline"},
+                ],
+                default="any",
+                description="Accepted phone number types",
+            ),
+        ],
+        tags=["localization", "chinese", "phone", "contact"],
+        severity_default="medium",
+    ),
+]

truthound_dashboard/schemas/validators/ml_feature_validators.py

@@ -0,0 +1,308 @@
+"""ML Feature validators.
+
+Validators for machine learning feature quality including null impact,
+scale validation, correlation analysis, and target leakage detection.
+
+Import path in truthound: `from truthound.validators.ml_feature import *`
+"""
+
+from .base import (
+    ParameterDefinition,
+    ParameterType,
+    ValidatorCategory,
+    ValidatorDefinition,
+)
+
+ML_FEATURE_VALIDATORS: list[ValidatorDefinition] = [
+    ValidatorDefinition(
+        name="FeatureNullImpact",
+        display_name="Feature Null Impact Analysis",
+        category=ValidatorCategory.ML_FEATURE,
+        description="Analyzes and validates the impact of null values on ML features.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Feature Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Feature column to analyze",
+            ),
+            ParameterDefinition(
+                name="target_column",
+                label="Target Column",
+                type=ParameterType.COLUMN,
+                description="Target column for impact analysis",
+            ),
+            ParameterDefinition(
+                name="max_null_ratio",
+                label="Maximum Null Ratio",
+                type=ParameterType.FLOAT,
+                default=0.1,
+                min_value=0,
+                max_value=1,
+                description="Maximum acceptable ratio of null values",
+            ),
+            ParameterDefinition(
+                name="max_impact_score",
+                label="Maximum Impact Score",
+                type=ParameterType.FLOAT,
+                min_value=0,
+                max_value=1,
+                description="Maximum acceptable impact score on target",
+            ),
+        ],
+        tags=["ml_feature", "null", "impact", "missing"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="FeatureScale",
+        display_name="Feature Scale Validation",
+        category=ValidatorCategory.ML_FEATURE,
+        description="Validates that features are properly scaled for ML algorithms.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Feature Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Feature column to validate",
+            ),
+            ParameterDefinition(
+                name="expected_scale",
+                label="Expected Scale Type",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "standard", "label": "Standard (mean=0, std=1)"},
+                    {"value": "minmax", "label": "Min-Max (0 to 1)"},
+                    {"value": "robust", "label": "Robust (median-centered)"},
+                    {"value": "unit_norm", "label": "Unit Norm (L2=1)"},
+                ],
+                required=True,
+                description="Expected feature scale type",
+            ),
+            ParameterDefinition(
+                name="tolerance",
+                label="Tolerance",
+                type=ParameterType.FLOAT,
+                default=0.1,
+                min_value=0,
+                max_value=1,
+                description="Acceptable deviation from expected scale",
+            ),
+        ],
+        tags=["ml_feature", "scale", "normalization", "preprocessing"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="FeatureCorrelationMatrix",
+        display_name="Feature Correlation Matrix",
+        category=ValidatorCategory.ML_FEATURE,
+        description="Validates correlation between features for multicollinearity detection.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Feature Columns",
+                type=ParameterType.COLUMN_LIST,
+                required=True,
+                description="Feature columns to analyze correlation",
+            ),
+            ParameterDefinition(
+                name="max_correlation",
+                label="Maximum Correlation",
+                type=ParameterType.FLOAT,
+                default=0.9,
+                min_value=0,
+                max_value=1,
+                description="Maximum acceptable absolute correlation between features",
+            ),
+            ParameterDefinition(
+                name="method",
+                label="Correlation Method",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "pearson", "label": "Pearson"},
+                    {"value": "spearman", "label": "Spearman"},
+                    {"value": "kendall", "label": "Kendall"},
+                ],
+                default="pearson",
+                description="Correlation calculation method",
+            ),
+        ],
+        tags=["ml_feature", "correlation", "multicollinearity", "redundancy"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="TargetLeakage",
+        display_name="Target Leakage Detection",
+        category=ValidatorCategory.ML_FEATURE,
+        description="Detects potential target leakage in features.",
+        parameters=[
+            ParameterDefinition(
+                name="feature_columns",
+                label="Feature Columns",
+                type=ParameterType.COLUMN_LIST,
+                required=True,
+                description="Feature columns to check for leakage",
+            ),
+            ParameterDefinition(
+                name="target_column",
+                label="Target Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Target column",
+            ),
+            ParameterDefinition(
+                name="leakage_threshold",
+                label="Leakage Threshold",
+                type=ParameterType.FLOAT,
+                default=0.95,
+                min_value=0.5,
+                max_value=1,
+                description="Correlation threshold indicating potential leakage",
+            ),
+        ],
+        tags=["ml_feature", "leakage", "target", "data_leakage"],
+        severity_default="critical",
+    ),
+    ValidatorDefinition(
+        name="FeatureImportance",
+        display_name="Feature Importance Threshold",
+        category=ValidatorCategory.ML_FEATURE,
+        description="Validates minimum feature importance for ML model utility.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Feature Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Feature column to validate",
+            ),
+            ParameterDefinition(
+                name="target_column",
+                label="Target Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Target column for importance calculation",
+            ),
+            ParameterDefinition(
+                name="min_importance",
+                label="Minimum Importance",
+                type=ParameterType.FLOAT,
+                default=0.01,
+                min_value=0,
+                max_value=1,
+                description="Minimum required feature importance score",
+            ),
+            ParameterDefinition(
+                name="method",
+                label="Importance Method",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "mutual_info", "label": "Mutual Information"},
+                    {"value": "correlation", "label": "Correlation"},
+                    {"value": "f_score", "label": "F-Score (ANOVA)"},
+                ],
+                default="mutual_info",
+                description="Method for calculating feature importance",
+            ),
+        ],
+        tags=["ml_feature", "importance", "relevance", "selection"],
+        severity_default="low",
+    ),
+    ValidatorDefinition(
+        name="FeatureVariance",
+        display_name="Feature Variance Threshold",
+        category=ValidatorCategory.ML_FEATURE,
+        description="Validates minimum feature variance (removes near-constant features).",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Feature Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Feature column to validate",
+            ),
+            ParameterDefinition(
+                name="min_variance",
+                label="Minimum Variance",
+                type=ParameterType.FLOAT,
+                default=0.01,
+                min_value=0,
+                description="Minimum required variance",
+            ),
+        ],
+        tags=["ml_feature", "variance", "constant", "low_variance"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="FeatureOutlierRatio",
+        display_name="Feature Outlier Ratio",
+        category=ValidatorCategory.ML_FEATURE,
+        description="Validates that outlier ratio in features is within acceptable bounds.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Feature Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Feature column to check",
+            ),
+            ParameterDefinition(
+                name="max_outlier_ratio",
+                label="Maximum Outlier Ratio",
+                type=ParameterType.FLOAT,
+                default=0.05,
+                min_value=0,
+                max_value=1,
+                description="Maximum acceptable ratio of outliers",
+            ),
+            ParameterDefinition(
+                name="method",
+                label="Outlier Detection Method",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "iqr", "label": "IQR (1.5x)"},
+                    {"value": "zscore", "label": "Z-Score (3 sigma)"},
+                    {"value": "mad", "label": "MAD"},
+                ],
+                default="iqr",
+                description="Method for outlier detection",
+            ),
+        ],
+        tags=["ml_feature", "outlier", "anomaly", "preprocessing"],
+        severity_default="medium",
+    ),
+    ValidatorDefinition(
+        name="ClassImbalance",
+        display_name="Class Imbalance Check",
+        category=ValidatorCategory.ML_FEATURE,
+        description="Validates class distribution for classification targets.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Target Column",
+                type=ParameterType.COLUMN,
+                required=True,
+                description="Classification target column",
+            ),
+            ParameterDefinition(
+                name="max_imbalance_ratio",
+                label="Maximum Imbalance Ratio",
+                type=ParameterType.FLOAT,
+                default=10,
+                min_value=1,
+                description="Maximum ratio of majority to minority class",
+            ),
+            ParameterDefinition(
+                name="min_samples_per_class",
+                label="Minimum Samples per Class",
+                type=ParameterType.INTEGER,
+                default=10,
+                min_value=1,
+                description="Minimum required samples for each class",
+            ),
+        ],
+        tags=["ml_feature", "class_imbalance", "target", "classification"],
+        severity_default="high",
+    ),
+]