truthound-dashboard 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

truthound_dashboard/schemas/validators/anomaly_validators.py

@@ -0,0 +1,723 @@
+"""Anomaly validators.
+
+ML-based outlier and anomaly detection validators.
+Requires: pip install truthound[anomaly] (scipy + scikit-learn)
+"""
+
+from .base import (
+    ParameterDefinition,
+    ParameterType,
+    ValidatorCategory,
+    ValidatorDefinition,
+)
+
+ANOMALY_VALIDATORS: list[ValidatorDefinition] = [
+    ValidatorDefinition(
+        name="IQRAnomaly",
+        display_name="IQR Anomaly",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects outliers using Interquartile Range (IQR) method.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="iqr_multiplier",
+                label="IQR Multiplier",
+                type=ParameterType.FLOAT,
+                description="Multiplier for IQR (1.5 = standard, 3.0 = extreme)",
+                default=1.5,
+                min_value=0,
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                description="Maximum acceptable anomaly ratio (0.0-1.0)",
+                default=0.05,
+                min_value=0,
+                max_value=1,
+            ),
+            ParameterDefinition(
+                name="detect_lower",
+                label="Detect Lower Outliers",
+                type=ParameterType.BOOLEAN,
+                default=True,
+            ),
+            ParameterDefinition(
+                name="detect_upper",
+                label="Detect Upper Outliers",
+                type=ParameterType.BOOLEAN,
+                default=True,
+            ),
+        ],
+        tags=["anomaly", "outlier", "iqr", "statistics"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="ZScoreAnomaly",
+        display_name="Z-Score Anomaly",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects outliers using Z-score methodology.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="threshold",
+                label="Z-Score Threshold",
+                type=ParameterType.FLOAT,
+                description="Z-score threshold (default: 3.0)",
+                default=3.0,
+                min_value=0,
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.05,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "outlier", "zscore", "statistics"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="MADAnomaly",
+        display_name="MAD Anomaly",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects outliers using Median Absolute Deviation.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="threshold",
+                label="MAD Threshold",
+                type=ParameterType.FLOAT,
+                description="MAD threshold (default: 3.5)",
+                default=3.5,
+                min_value=0,
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.05,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "outlier", "mad", "robust"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="IsolationForest",
+        display_name="Isolation Forest",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects anomalies using Isolation Forest algorithm.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns to analyze (leave empty for all numeric)",
+            ),
+            ParameterDefinition(
+                name="contamination",
+                label="Contamination",
+                type=ParameterType.FLOAT,
+                description="Expected proportion of outliers (0.0-0.5, or 'auto')",
+                default=0.1,
+                min_value=0,
+                max_value=0.5,
+            ),
+            ParameterDefinition(
+                name="n_estimators",
+                label="Number of Estimators",
+                type=ParameterType.INTEGER,
+                description="Number of base estimators in the ensemble",
+                default=100,
+                min_value=1,
+            ),
+            ParameterDefinition(
+                name="max_samples",
+                label="Max Samples",
+                type=ParameterType.INTEGER,
+                description="Number of samples to draw (leave empty for 256)",
+            ),
+            ParameterDefinition(
+                name="random_state",
+                label="Random State",
+                type=ParameterType.INTEGER,
+                description="Random seed for reproducibility",
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.1,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "isolation_forest", "machine_learning", "multivariate"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="LOF",
+        display_name="Local Outlier Factor",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects anomalies using Local Outlier Factor algorithm.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns to analyze",
+            ),
+            ParameterDefinition(
+                name="n_neighbors",
+                label="Number of Neighbors",
+                type=ParameterType.INTEGER,
+                description="Number of neighbors to use",
+                default=20,
+                min_value=1,
+            ),
+            ParameterDefinition(
+                name="contamination",
+                label="Contamination",
+                type=ParameterType.FLOAT,
+                description="Expected proportion of outliers",
+                default=0.1,
+                min_value=0,
+                max_value=0.5,
+            ),
+            ParameterDefinition(
+                name="metric",
+                label="Distance Metric",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "euclidean", "label": "Euclidean"},
+                    {"value": "manhattan", "label": "Manhattan"},
+                    {"value": "minkowski", "label": "Minkowski"},
+                    {"value": "cosine", "label": "Cosine"},
+                ],
+                default="euclidean",
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.1,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "lof", "machine_learning", "density"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="DBSCAN",
+        display_name="DBSCAN Anomaly",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects anomalies using DBSCAN clustering (noise points).",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns to analyze",
+            ),
+            ParameterDefinition(
+                name="eps",
+                label="Epsilon (eps)",
+                type=ParameterType.FLOAT,
+                description="Maximum distance between samples in a neighborhood",
+                default=0.5,
+                min_value=0,
+            ),
+            ParameterDefinition(
+                name="min_samples",
+                label="Min Samples",
+                type=ParameterType.INTEGER,
+                description="Minimum samples in a neighborhood for a core point",
+                default=5,
+                min_value=1,
+            ),
+            ParameterDefinition(
+                name="metric",
+                label="Distance Metric",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "euclidean", "label": "Euclidean"},
+                    {"value": "manhattan", "label": "Manhattan"},
+                    {"value": "cosine", "label": "Cosine"},
+                ],
+                default="euclidean",
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.1,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "dbscan", "clustering", "density"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="Mahalanobis",
+        display_name="Mahalanobis Distance",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects multivariate outliers using Mahalanobis distance.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns to analyze (leave empty for all numeric)",
+            ),
+            ParameterDefinition(
+                name="threshold_percentile",
+                label="Threshold Percentile",
+                type=ParameterType.FLOAT,
+                description="Chi-squared percentile threshold (e.g., 97.5)",
+                default=97.5,
+                min_value=0,
+                max_value=100,
+            ),
+            ParameterDefinition(
+                name="use_robust_covariance",
+                label="Use Robust Covariance",
+                type=ParameterType.BOOLEAN,
+                description="Use Minimum Covariance Determinant (more robust)",
+                default=True,
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.05,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "mahalanobis", "multivariate", "statistics"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="OneClassSVM",
+        display_name="One-Class SVM",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects anomalies using One-Class SVM.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns to analyze",
+            ),
+            ParameterDefinition(
+                name="kernel",
+                label="Kernel",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "rbf", "label": "RBF (Radial Basis Function)"},
+                    {"value": "linear", "label": "Linear"},
+                    {"value": "poly", "label": "Polynomial"},
+                    {"value": "sigmoid", "label": "Sigmoid"},
+                ],
+                default="rbf",
+            ),
+            ParameterDefinition(
+                name="nu",
+                label="Nu",
+                type=ParameterType.FLOAT,
+                description="Upper bound on the fraction of margin errors (0.0-1.0)",
+                default=0.1,
+                min_value=0,
+                max_value=1,
+            ),
+            ParameterDefinition(
+                name="gamma",
+                label="Gamma",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "scale", "label": "Scale (1 / (n_features * X.var()))"},
+                    {"value": "auto", "label": "Auto (1 / n_features)"},
+                ],
+                default="scale",
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.1,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "svm", "machine_learning"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="EllipticEnvelope",
+        display_name="Elliptic Envelope",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects outliers using robust Gaussian fitting.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns to analyze",
+            ),
+            ParameterDefinition(
+                name="contamination",
+                label="Contamination",
+                type=ParameterType.FLOAT,
+                description="Expected proportion of outliers",
+                default=0.1,
+                min_value=0,
+                max_value=0.5,
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.1,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "elliptic", "gaussian", "covariance"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="PCAnomaly",
+        display_name="PCA Anomaly",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects anomalies using PCA reconstruction error.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns to analyze",
+            ),
+            ParameterDefinition(
+                name="n_components",
+                label="Number of Components",
+                type=ParameterType.INTEGER,
+                description="Number of principal components to use",
+            ),
+            ParameterDefinition(
+                name="variance_ratio",
+                label="Variance Ratio",
+                type=ParameterType.FLOAT,
+                description="Target explained variance ratio (0.0-1.0)",
+                default=0.95,
+                min_value=0,
+                max_value=1,
+            ),
+            ParameterDefinition(
+                name="error_percentile",
+                label="Error Percentile",
+                type=ParameterType.FLOAT,
+                description="Reconstruction error percentile threshold",
+                default=95,
+                min_value=0,
+                max_value=100,
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.05,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "pca", "dimensionality", "reconstruction"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="GrubbsTest",
+        display_name="Grubbs Test",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects outliers using Grubbs statistical test.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="alpha",
+                label="Significance Level",
+                type=ParameterType.FLOAT,
+                description="Significance level for the test",
+                default=0.05,
+                min_value=0,
+                max_value=1,
+            ),
+            ParameterDefinition(
+                name="max_iterations",
+                label="Max Iterations",
+                type=ParameterType.INTEGER,
+                description="Maximum number of outliers to remove",
+                default=10,
+                min_value=1,
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.05,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "grubbs", "statistical", "test"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="TukeyFences",
+        display_name="Tukey Fences",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects outliers using Tukey's inner and outer fences.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="detect_mild",
+                label="Detect Mild Outliers",
+                type=ParameterType.BOOLEAN,
+                description="Detect values outside inner fences (1.5 * IQR)",
+                default=True,
+            ),
+            ParameterDefinition(
+                name="detect_extreme",
+                label="Detect Extreme Outliers",
+                type=ParameterType.BOOLEAN,
+                description="Detect values outside outer fences (3.0 * IQR)",
+                default=True,
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.05,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "tukey", "fences", "statistics"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="PercentileAnomaly",
+        display_name="Percentile Anomaly",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects outliers based on percentile thresholds.",
+        parameters=[
+            ParameterDefinition(
+                name="column",
+                label="Column",
+                type=ParameterType.COLUMN,
+                required=True,
+            ),
+            ParameterDefinition(
+                name="lower_percentile",
+                label="Lower Percentile",
+                type=ParameterType.FLOAT,
+                description="Lower percentile threshold",
+                default=1.0,
+                min_value=0,
+                max_value=100,
+            ),
+            ParameterDefinition(
+                name="upper_percentile",
+                label="Upper Percentile",
+                type=ParameterType.FLOAT,
+                description="Upper percentile threshold",
+                default=99.0,
+                min_value=0,
+                max_value=100,
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.02,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "percentile", "threshold"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="ZScoreMultivariate",
+        display_name="Multivariate Z-Score",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects multivariate outliers using combined Z-scores.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns to analyze",
+            ),
+            ParameterDefinition(
+                name="threshold",
+                label="Z-Score Threshold",
+                type=ParameterType.FLOAT,
+                default=3.0,
+                min_value=0,
+            ),
+            ParameterDefinition(
+                name="method",
+                label="Combination Method",
+                type=ParameterType.SELECT,
+                options=[
+                    {"value": "any", "label": "Any (outlier in any column)"},
+                    {"value": "all", "label": "All (outlier in all columns)"},
+                    {"value": "mean", "label": "Mean (average Z-score)"},
+                    {"value": "max", "label": "Max (maximum Z-score)"},
+                ],
+                default="any",
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.05,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "zscore", "multivariate"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+    ValidatorDefinition(
+        name="AutoEncoder",
+        display_name="AutoEncoder Anomaly",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects anomalies using neural network autoencoder reconstruction error.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns to analyze",
+            ),
+            ParameterDefinition(
+                name="encoding_dim",
+                label="Encoding Dimension",
+                type=ParameterType.INTEGER,
+                description="Dimensionality of the encoded representation",
+            ),
+            ParameterDefinition(
+                name="epochs",
+                label="Training Epochs",
+                type=ParameterType.INTEGER,
+                default=50,
+                min_value=1,
+            ),
+            ParameterDefinition(
+                name="error_percentile",
+                label="Error Percentile",
+                type=ParameterType.FLOAT,
+                default=95,
+                min_value=0,
+                max_value=100,
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.05,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "autoencoder", "deep_learning", "neural_network"],
+        severity_default="medium",
+        requires_extra="anomaly",
+        experimental=True,
+    ),
+    ValidatorDefinition(
+        name="KMeansAnomaly",
+        display_name="K-Means Anomaly",
+        category=ValidatorCategory.ANOMALY,
+        description="Detects anomalies based on distance from K-Means cluster centers.",
+        parameters=[
+            ParameterDefinition(
+                name="columns",
+                label="Columns",
+                type=ParameterType.COLUMN_LIST,
+                description="Columns to analyze",
+            ),
+            ParameterDefinition(
+                name="n_clusters",
+                label="Number of Clusters",
+                type=ParameterType.INTEGER,
+                default=5,
+                min_value=2,
+            ),
+            ParameterDefinition(
+                name="distance_percentile",
+                label="Distance Percentile",
+                type=ParameterType.FLOAT,
+                description="Percentile threshold for anomaly detection",
+                default=95,
+                min_value=0,
+                max_value=100,
+            ),
+            ParameterDefinition(
+                name="max_anomaly_ratio",
+                label="Max Anomaly Ratio",
+                type=ParameterType.FLOAT,
+                default=0.05,
+                min_value=0,
+                max_value=1,
+            ),
+        ],
+        tags=["anomaly", "kmeans", "clustering"],
+        severity_default="medium",
+        requires_extra="anomaly",
+    ),
+]