validmind 2.8.29__py3-none-any.whl → 2.9.1__py3-none-any.whl

validmind/tests/stats/OutlierDetection.py

@@ -0,0 +1,173 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+from sklearn.ensemble import IsolationForest
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.utils import format_records
+from validmind.vm_models import VMDataset
+
+
+def _validate_columns(dataset: VMDataset, columns: Optional[List[str]]):
+    """Validate and return numerical columns."""
+    if columns is None:
+        columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        columns = [col for col in columns if col in available_columns]
+
+    # Filter out boolean columns as they can't be used for outlier detection
+    numeric_columns = []
+    for col in columns:
+        if col in dataset.df.columns:
+            col_dtype = dataset.df[col].dtype
+            # Exclude boolean and object types, keep only true numeric types
+            if pd.api.types.is_numeric_dtype(col_dtype) and col_dtype != bool:
+                numeric_columns.append(col)
+
+    columns = numeric_columns
+
+    if not columns:
+        raise SkipTestError("No suitable numerical columns found for outlier detection")
+
+    return columns
+
+
+def _detect_iqr_outliers(data, iqr_threshold: float):
+    """Detect outliers using IQR method."""
+    q1, q3 = data.quantile(0.25), data.quantile(0.75)
+    iqr = q3 - q1
+    lower_bound = q1 - iqr_threshold * iqr
+    upper_bound = q3 + iqr_threshold * iqr
+    # Fix numpy boolean operation error by using pandas boolean indexing properly
+    outlier_mask = (data < lower_bound) | (data > upper_bound)
+    iqr_outliers = data[outlier_mask]
+    return len(iqr_outliers), (len(iqr_outliers) / len(data)) * 100
+
+
+def _detect_zscore_outliers(data, zscore_threshold: float):
+    """Detect outliers using Z-score method."""
+    z_scores = np.abs(stats.zscore(data))
+    # Fix potential numpy boolean operation error
+    outlier_mask = z_scores > zscore_threshold
+    zscore_outliers = data[outlier_mask]
+    return len(zscore_outliers), (len(zscore_outliers) / len(data)) * 100
+
+
+def _detect_isolation_forest_outliers(data, contamination: float):
+    """Detect outliers using Isolation Forest method."""
+    if len(data) <= 10:
+        return 0, 0
+
+    try:
+        iso_forest = IsolationForest(contamination=contamination, random_state=42)
+        outlier_pred = iso_forest.fit_predict(data.values.reshape(-1, 1))
+        iso_outliers = data[outlier_pred == -1]
+        return len(iso_outliers), (len(iso_outliers) / len(data)) * 100
+    except Exception:
+        return 0, 0
+
+
+def _process_column_outliers(
+    column: str,
+    data,
+    methods: List[str],
+    iqr_threshold: float,
+    zscore_threshold: float,
+    contamination: float,
+):
+    """Process outlier detection for a single column."""
+    outliers_dict = {"Feature": column, "Total Count": len(data)}
+
+    # IQR method
+    if "iqr" in methods:
+        count, percentage = _detect_iqr_outliers(data, iqr_threshold)
+        outliers_dict["IQR Outliers"] = count
+        outliers_dict["IQR %"] = percentage
+
+    # Z-score method
+    if "zscore" in methods:
+        count, percentage = _detect_zscore_outliers(data, zscore_threshold)
+        outliers_dict["Z-Score Outliers"] = count
+        outliers_dict["Z-Score %"] = percentage
+
+    # Isolation Forest method
+    if "isolation_forest" in methods:
+        count, percentage = _detect_isolation_forest_outliers(data, contamination)
+        outliers_dict["Isolation Forest Outliers"] = count
+        outliers_dict["Isolation Forest %"] = percentage
+
+    return outliers_dict
+
+
+@tags("tabular_data", "statistics", "outliers")
+@tasks("classification", "regression", "clustering")
+def OutlierDetection(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    methods: List[str] = ["iqr", "zscore", "isolation_forest"],
+    iqr_threshold: float = 1.5,
+    zscore_threshold: float = 3.0,
+    contamination: float = 0.1,
+) -> Dict[str, Any]:
+    """
+    Detects outliers in numerical features using multiple statistical methods.
+
+    ### Purpose
+
+    This test identifies outliers in numerical features using various statistical
+    methods including IQR, Z-score, and Isolation Forest. It provides comprehensive
+    outlier detection to help identify data quality issues and potential anomalies.
+
+    ### Test Mechanism
+
+    The test applies multiple outlier detection methods:
+    - IQR method: Values beyond Q1 - 1.5*IQR or Q3 + 1.5*IQR
+    - Z-score method: Values with |z-score| > threshold
+    - Isolation Forest: ML-based anomaly detection
+
+    ### Signs of High Risk
+
+    - High percentage of outliers indicating data quality issues
+    - Inconsistent outlier detection across methods
+    - Extreme outliers that significantly deviate from normal patterns
+
+    ### Strengths
+
+    - Multiple detection methods for robust outlier identification
+    - Customizable thresholds for different sensitivity levels
+    - Clear summary of outlier patterns across features
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Some methods assume normal distributions
+    - Threshold selection can be subjective
+    """
+    # Validate inputs
+    columns = _validate_columns(dataset, columns)
+
+    # Process each column
+    outlier_summary = []
+    for column in columns:
+        data = dataset._df[column].dropna()
+
+        if len(data) >= 3:
+            outliers_dict = _process_column_outliers(
+                column, data, methods, iqr_threshold, zscore_threshold, contamination
+            )
+            outlier_summary.append(outliers_dict)
+
+    # Format results
+    results = {}
+    if outlier_summary:
+        results["Outlier Summary"] = format_records(pd.DataFrame(outlier_summary))
+
+    return results

validmind/unit_metrics/classification/individual/AbsoluteError.py

@@ -0,0 +1,42 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def AbsoluteError(model: VMModel, dataset: VMDataset, **kwargs) -> List[float]:
+    """Calculates the absolute error per row for a classification model.
+
+    For classification tasks, this computes the absolute difference between
+    the true class labels and predicted class labels for each individual row.
+    For binary classification with probabilities, it can also compute the
+    absolute difference between true labels and predicted probabilities.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predictions
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row absolute errors as a list of float values
+    """
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Convert to numpy arrays and ensure same data type
+    y_true = np.asarray(y_true)
+    y_pred = np.asarray(y_pred)
+
+    # For classification, compute absolute difference between true and predicted labels
+    absolute_errors = np.abs(y_true - y_pred)
+
+    # Return as a list of floats
+    return absolute_errors.astype(float).tolist()

validmind/unit_metrics/classification/individual/BrierScore.py

@@ -0,0 +1,56 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def BrierScore(model: VMModel, dataset: VMDataset, **kwargs) -> List[float]:
+    """Calculates the Brier score per row for a classification model.
+
+    The Brier score is a proper score function that measures the accuracy of
+    probabilistic predictions. It is calculated as the mean squared difference
+    between predicted probabilities and the actual binary outcomes.
+    Lower scores indicate better calibration.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predicted probabilities
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row Brier scores as a list of float values
+
+    Raises:
+        ValueError: If probability column is not found for the model
+    """
+    y_true = dataset.y
+
+    # Try to get probabilities
+    try:
+        y_prob = dataset.y_prob(model)
+        # For binary classification, use the positive class probability
+        if y_prob.ndim > 1 and y_prob.shape[1] > 1:
+            y_prob = y_prob[:, 1]  # Use probability of positive class
+    except ValueError:
+        # Fall back to predictions if probabilities not available
+        # Convert predictions to "probabilities" (1.0 for predicted class, 0.0 for other)
+        y_pred = dataset.y_pred(model)
+        y_prob = y_pred.astype(float)
+
+    # Convert to numpy arrays and ensure same data type
+    y_true = np.asarray(y_true, dtype=float)
+    y_prob = np.asarray(y_prob, dtype=float)
+
+    # Calculate Brier score per row: (predicted_probability - actual_outcome)²
+    brier_scores = (y_prob - y_true) ** 2
+
+    # Return as a list of floats
+    return brier_scores.tolist()

validmind/unit_metrics/classification/individual/CalibrationError.py

@@ -0,0 +1,77 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def CalibrationError(
+    model: VMModel, dataset: VMDataset, n_bins: int = 10, **kwargs
+) -> List[float]:
+    """Calculates the calibration error per row for a classification model.
+
+    Calibration error measures how well the predicted probabilities reflect the
+    actual likelihood of the positive class. For each prediction, this computes
+    the absolute difference between the predicted probability and the empirical
+    frequency of the positive class in the corresponding probability bin.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predicted probabilities
+        n_bins: Number of bins for probability calibration, defaults to 10
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row calibration errors as a list of float values
+
+    Raises:
+        ValueError: If probability column is not found for the model
+    """
+    y_true = dataset.y
+
+    # Try to get probabilities
+    try:
+        y_prob = dataset.y_prob(model)
+        # For binary classification, use the positive class probability
+        if y_prob.ndim > 1 and y_prob.shape[1] > 1:
+            y_prob = y_prob[:, 1]  # Use probability of positive class
+    except ValueError:
+        # If no probabilities available, return zeros (perfect calibration for hard predictions)
+        return [0.0] * len(y_true)
+
+    # Convert to numpy arrays
+    y_true = np.asarray(y_true, dtype=float)
+    y_prob = np.asarray(y_prob, dtype=float)
+
+    # Create probability bins
+    bin_boundaries = np.linspace(0, 1, n_bins + 1)
+    bin_lowers = bin_boundaries[:-1]
+    bin_uppers = bin_boundaries[1:]
+
+    # Calculate calibration error for each sample
+    calibration_errors = np.zeros_like(y_prob)
+
+    for bin_lower, bin_upper in zip(bin_lowers, bin_uppers):
+        # Find samples in this bin
+        in_bin = (y_prob > bin_lower) & (y_prob <= bin_upper)
+        if not np.any(in_bin):
+            continue
+
+        # Calculate empirical frequency for this bin
+        empirical_freq = np.mean(y_true[in_bin])
+
+        # Calculate average predicted probability for this bin
+        avg_predicted_prob = np.mean(y_prob[in_bin])
+
+        # Assign calibration error to all samples in this bin
+        calibration_errors[in_bin] = abs(avg_predicted_prob - empirical_freq)
+
+    # Return as a list of floats
+    return calibration_errors.tolist()

validmind/unit_metrics/classification/individual/ClassBalance.py

@@ -0,0 +1,65 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def ClassBalance(model: VMModel, dataset: VMDataset, **kwargs) -> List[float]:
+    """Calculates the class balance score per row for a classification model.
+
+    For each prediction, this returns how balanced the predicted class is in the
+    training distribution. Lower scores indicate predictions on rare classes,
+    higher scores indicate predictions on common classes. This helps understand
+    if model errors are more likely on imbalanced classes.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predictions
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row class balance scores as a list of float values
+
+    Note:
+        Scores range from 0 to 0.5, where 0.5 indicates perfectly balanced classes
+        and lower values indicate more imbalanced classes.
+    """
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Convert to numpy arrays
+    y_true = np.asarray(y_true)
+    y_pred = np.asarray(y_pred)
+
+    # Calculate class frequencies in the true labels (proxy for training distribution)
+    unique_classes, class_counts = np.unique(y_true, return_counts=True)
+    class_frequencies = class_counts / len(y_true)
+
+    # Create a mapping from class to frequency
+    class_to_freq = dict(zip(unique_classes, class_frequencies))
+
+    # Calculate balance score for each prediction
+    balance_scores = []
+
+    for pred in y_pred:
+        if pred in class_to_freq:
+            freq = class_to_freq[pred]
+            # Balance score: how close to 0.5 (perfectly balanced) the frequency is
+            # Score = 0.5 - |freq - 0.5| = min(freq, 1-freq)
+            balance_score = min(freq, 1 - freq)
+        else:
+            # Predicted class not seen in true labels (very rare)
+            balance_score = 0.0
+
+        balance_scores.append(balance_score)
+
+    # Return as a list of floats
+    return balance_scores

validmind/unit_metrics/classification/individual/Confidence.py

@@ -0,0 +1,52 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def Confidence(model: VMModel, dataset: VMDataset, **kwargs) -> List[float]:
+    """Calculates the prediction confidence per row for a classification model.
+
+    For binary classification, confidence is calculated as the maximum probability
+    across classes, or alternatively as the distance from the decision boundary (0.5).
+    Higher values indicate more confident predictions.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predicted probabilities
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row confidence scores as a list of float values
+
+    Raises:
+        ValueError: If probability column is not found for the model
+    """
+    # Try to get probabilities, fall back to predictions if not available
+    try:
+        y_prob = dataset.y_prob(model)
+        # For binary classification, use max probability approach
+        if y_prob.ndim > 1 and y_prob.shape[1] > 1:
+            # Multi-class: confidence is the maximum probability
+            confidence = np.max(y_prob, axis=1)
+        else:
+            # Binary classification: confidence based on distance from 0.5
+            y_prob = np.asarray(y_prob, dtype=float)
+            confidence = np.abs(y_prob - 0.5) + 0.5
+    except ValueError:
+        # Fall back to binary correctness if probabilities not available
+        y_true = dataset.y
+        y_pred = dataset.y_pred(model)
+        # If no probabilities, confidence is 1.0 for correct, 0.0 for incorrect
+        confidence = (y_true == y_pred).astype(float)
+
+    # Return as a list of floats
+    return confidence.tolist()

validmind/unit_metrics/classification/individual/Correctness.py

@@ -0,0 +1,41 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def Correctness(model: VMModel, dataset: VMDataset, **kwargs) -> List[int]:
+    """Calculates the correctness per row for a classification model.
+
+    For classification tasks, this returns 1 for correctly classified rows
+    and 0 for incorrectly classified rows. This provides a binary indicator
+    of model performance for each individual prediction.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predictions
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[int]: Per-row correctness as a list of 1s and 0s
+    """
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Convert to numpy arrays
+    y_true = np.asarray(y_true)
+    y_pred = np.asarray(y_pred)
+
+    # For classification, check if predictions match true labels
+    correctness = (y_true == y_pred).astype(int)
+
+    # Return as a list of integers
+    return correctness.tolist()

validmind/unit_metrics/classification/individual/LogLoss.py

@@ -0,0 +1,61 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def LogLoss(
+    model: VMModel, dataset: VMDataset, eps: float = 1e-15, **kwargs
+) -> List[float]:
+    """Calculates the logarithmic loss per row for a classification model.
+
+    Log loss measures the performance of a classification model where the prediction
+    is a probability value between 0 and 1. The log loss increases as the predicted
+    probability diverges from the actual label.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predicted probabilities
+        eps: Small value to avoid log(0), defaults to 1e-15
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row log loss values as a list of float values
+
+    Raises:
+        ValueError: If probability column is not found for the model
+    """
+    y_true = dataset.y
+
+    # Try to get probabilities
+    try:
+        y_prob = dataset.y_prob(model)
+        # For binary classification, use the positive class probability
+        if y_prob.ndim > 1 and y_prob.shape[1] > 1:
+            y_prob = y_prob[:, 1]  # Use probability of positive class
+    except ValueError:
+        # Fall back to predictions if probabilities not available
+        # Convert predictions to "probabilities" (0.99 for correct class, 0.01 for wrong)
+        y_pred = dataset.y_pred(model)
+        y_prob = np.where(y_true == y_pred, 0.99, 0.01)
+
+    # Convert to numpy arrays and ensure same data type
+    y_true = np.asarray(y_true, dtype=float)
+    y_prob = np.asarray(y_prob, dtype=float)
+
+    # Clip probabilities to avoid log(0) and log(1)
+    y_prob = np.clip(y_prob, eps, 1 - eps)
+
+    # Calculate log loss per row: -[y*log(p) + (1-y)*log(1-p)]
+    log_loss_per_row = -(y_true * np.log(y_prob) + (1 - y_true) * np.log(1 - y_prob))
+
+    # Return as a list of floats
+    return log_loss_per_row.tolist()

validmind/unit_metrics/classification/individual/OutlierScore.py

@@ -0,0 +1,86 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+from sklearn.ensemble import IsolationForest
+from sklearn.preprocessing import StandardScaler
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def OutlierScore(
+    model: VMModel, dataset: VMDataset, contamination: float = 0.1, **kwargs
+) -> List[float]:
+    """Calculates the outlier score per row for a classification model.
+
+    Uses Isolation Forest to identify samples that deviate significantly from
+    the typical patterns in the feature space. Higher scores indicate more
+    anomalous/outlier-like samples. This can help identify out-of-distribution
+    samples or data points that might be harder to predict accurately.
+
+    Args:
+        model: The classification model to evaluate (unused but kept for consistency)
+        dataset: The dataset containing feature data
+        contamination: Expected proportion of outliers, defaults to 0.1
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row outlier scores as a list of float values
+
+    Note:
+        Scores are normalized to [0, 1] where higher values indicate more outlier-like samples
+    """
+    # Get feature data
+    X = dataset.x_df()
+
+    # Handle case where we have no features or only categorical features
+    if X.empty or X.shape[1] == 0:
+        # Return zero outlier scores if no features available
+        return [0.0] * len(dataset.y)
+
+    # Select only numeric features for outlier detection
+    numeric_features = dataset.feature_columns_numeric
+    if not numeric_features:
+        # If no numeric features, return zero outlier scores
+        return [0.0] * len(dataset.y)
+
+    X_numeric = X[numeric_features]
+
+    # Handle missing values by filling with median
+    X_filled = X_numeric.fillna(X_numeric.median())
+
+    # Standardize features for better outlier detection
+    scaler = StandardScaler()
+    X_scaled = scaler.fit_transform(X_filled)
+
+    # Fit Isolation Forest
+    isolation_forest = IsolationForest(
+        contamination=contamination, random_state=42, n_estimators=100
+    )
+
+    # Fit the model on the data
+    isolation_forest.fit(X_scaled)
+
+    # Get anomaly scores (negative values for outliers)
+    anomaly_scores = isolation_forest.decision_function(X_scaled)
+
+    # Convert to outlier scores (0 to 1, where 1 is most outlier-like)
+    # Normalize using min-max scaling
+    min_score = np.min(anomaly_scores)
+    max_score = np.max(anomaly_scores)
+
+    if max_score == min_score:
+        # All samples have same score, no outliers detected
+        outlier_scores = np.zeros_like(anomaly_scores)
+    else:
+        # Invert and normalize: higher values = more outlier-like
+        outlier_scores = (max_score - anomaly_scores) / (max_score - min_score)
+
+    # Return as a list of floats
+    return outlier_scores.tolist()