validmind 2.8.28__py3-none-any.whl → 2.9.1__py3-none-any.whl

validmind/unit_metrics/classification/individual/OutlierScore.py

@@ -0,0 +1,86 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+from sklearn.ensemble import IsolationForest
+from sklearn.preprocessing import StandardScaler
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def OutlierScore(
+    model: VMModel, dataset: VMDataset, contamination: float = 0.1, **kwargs
+) -> List[float]:
+    """Calculates the outlier score per row for a classification model.
+
+    Uses Isolation Forest to identify samples that deviate significantly from
+    the typical patterns in the feature space. Higher scores indicate more
+    anomalous/outlier-like samples. This can help identify out-of-distribution
+    samples or data points that might be harder to predict accurately.
+
+    Args:
+        model: The classification model to evaluate (unused but kept for consistency)
+        dataset: The dataset containing feature data
+        contamination: Expected proportion of outliers, defaults to 0.1
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row outlier scores as a list of float values
+
+    Note:
+        Scores are normalized to [0, 1] where higher values indicate more outlier-like samples
+    """
+    # Get feature data
+    X = dataset.x_df()
+
+    # Handle case where we have no features or only categorical features
+    if X.empty or X.shape[1] == 0:
+        # Return zero outlier scores if no features available
+        return [0.0] * len(dataset.y)
+
+    # Select only numeric features for outlier detection
+    numeric_features = dataset.feature_columns_numeric
+    if not numeric_features:
+        # If no numeric features, return zero outlier scores
+        return [0.0] * len(dataset.y)
+
+    X_numeric = X[numeric_features]
+
+    # Handle missing values by filling with median
+    X_filled = X_numeric.fillna(X_numeric.median())
+
+    # Standardize features for better outlier detection
+    scaler = StandardScaler()
+    X_scaled = scaler.fit_transform(X_filled)
+
+    # Fit Isolation Forest
+    isolation_forest = IsolationForest(
+        contamination=contamination, random_state=42, n_estimators=100
+    )
+
+    # Fit the model on the data
+    isolation_forest.fit(X_scaled)
+
+    # Get anomaly scores (negative values for outliers)
+    anomaly_scores = isolation_forest.decision_function(X_scaled)
+
+    # Convert to outlier scores (0 to 1, where 1 is most outlier-like)
+    # Normalize using min-max scaling
+    min_score = np.min(anomaly_scores)
+    max_score = np.max(anomaly_scores)
+
+    if max_score == min_score:
+        # All samples have same score, no outliers detected
+        outlier_scores = np.zeros_like(anomaly_scores)
+    else:
+        # Invert and normalize: higher values = more outlier-like
+        outlier_scores = (max_score - anomaly_scores) / (max_score - min_score)
+
+    # Return as a list of floats
+    return outlier_scores.tolist()

validmind/unit_metrics/classification/individual/ProbabilityError.py

@@ -0,0 +1,54 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def ProbabilityError(model: VMModel, dataset: VMDataset, **kwargs) -> List[float]:
+    """Calculates the probability error per row for a classification model.
+
+    For binary classification tasks, this computes the absolute difference between
+    the true class labels (0 or 1) and the predicted probabilities for each row.
+    This provides insight into how confident the model's predictions are and
+    how far off they are from the actual labels.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predicted probabilities
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row probability errors as a list of float values
+
+    Raises:
+        ValueError: If probability column is not found for the model
+    """
+    y_true = dataset.y
+
+    # Try to get probabilities, fall back to predictions if not available
+    try:
+        y_prob = dataset.y_prob(model)
+        # For binary classification, use the positive class probability
+        if y_prob.ndim > 1 and y_prob.shape[1] > 1:
+            y_prob = y_prob[:, 1]  # Use probability of positive class
+    except ValueError:
+        # Fall back to predictions if probabilities not available
+        y_prob = dataset.y_pred(model)
+
+    # Convert to numpy arrays and ensure same data type
+    y_true = np.asarray(y_true, dtype=float)
+    y_prob = np.asarray(y_prob, dtype=float)
+
+    # Compute absolute difference between true labels and predicted probabilities
+    probability_errors = np.abs(y_true - y_prob)
+
+    # Return as a list of floats
+    return probability_errors.tolist()

validmind/unit_metrics/classification/individual/Uncertainty.py

@@ -0,0 +1,60 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def Uncertainty(model: VMModel, dataset: VMDataset, **kwargs) -> List[float]:
+    """Calculates the prediction uncertainty per row for a classification model.
+
+    Uncertainty is measured using the entropy of the predicted probability distribution.
+    Higher entropy indicates higher uncertainty in the prediction. For binary
+    classification, maximum uncertainty occurs at probability 0.5.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predicted probabilities
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row uncertainty scores as a list of float values
+
+    Raises:
+        ValueError: If probability column is not found for the model
+    """
+    # Try to get probabilities
+    try:
+        y_prob = dataset.y_prob(model)
+
+        if y_prob.ndim > 1 and y_prob.shape[1] > 1:
+            # Multi-class: calculate entropy across all classes
+            # Clip to avoid log(0)
+            y_prob_clipped = np.clip(y_prob, 1e-15, 1 - 1e-15)
+            # Entropy: -sum(p * log(p))
+            uncertainty = -np.sum(y_prob_clipped * np.log(y_prob_clipped), axis=1)
+        else:
+            # Binary classification: calculate binary entropy
+            y_prob = np.asarray(y_prob, dtype=float)
+            # Clip to avoid log(0)
+            y_prob_clipped = np.clip(y_prob, 1e-15, 1 - 1e-15)
+            # Binary entropy: -[p*log(p) + (1-p)*log(1-p)]
+            uncertainty = -(
+                y_prob_clipped * np.log(y_prob_clipped)
+                + (1 - y_prob_clipped) * np.log(1 - y_prob_clipped)
+            )
+
+    except ValueError:
+        # If no probabilities available, assume zero uncertainty for hard predictions
+        n_samples = len(dataset.y)
+        uncertainty = np.zeros(n_samples)
+
+    # Return as a list of floats
+    return uncertainty.tolist()

validmind/unit_metrics/regression/AdjustedRSquaredScore.py

@@ -5,11 +5,12 @@
 from sklearn.metrics import r2_score as _r2_score
 
 from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("regression")
 @tasks("regression")
-def AdjustedRSquaredScore(model, dataset):
+def AdjustedRSquaredScore(model: VMModel, dataset: VMDataset) -> float:
     """Calculates the adjusted R-squared score for a regression model."""
     r2_score = _r2_score(
         dataset.y,

validmind/unit_metrics/regression/GiniCoefficient.py

@@ -5,11 +5,12 @@
 import numpy as np
 
 from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("regression")
 @tasks("regression")
-def GiniCoefficient(dataset, model):
+def GiniCoefficient(dataset: VMDataset, model: VMModel) -> float:
     """Calculates the Gini coefficient for a regression model."""
     y_true = dataset.y
     y_pred = dataset.y_pred(model)

validmind/unit_metrics/regression/HuberLoss.py

@@ -5,11 +5,12 @@
 import numpy as np
 
 from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("regression")
 @tasks("regression")
-def HuberLoss(model, dataset):
+def HuberLoss(model: VMModel, dataset: VMDataset) -> float:
     """Calculates the Huber loss for a regression model."""
     y_true = dataset.y
     y_pred = dataset.y_pred(model)

validmind/unit_metrics/regression/KolmogorovSmirnovStatistic.py

@@ -5,11 +5,12 @@
 import numpy as np
 
 from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("regression")
 @tasks("regression")
-def KolmogorovSmirnovStatistic(dataset, model):
+def KolmogorovSmirnovStatistic(dataset: VMDataset, model: VMModel) -> float:
     """Calculates the Kolmogorov-Smirnov statistic for a regression model."""
     y_true = dataset.y.flatten()
     y_pred = dataset.y_pred(model)

validmind/unit_metrics/regression/MeanAbsoluteError.py

@@ -5,10 +5,11 @@
 from sklearn.metrics import mean_absolute_error as _mean_absolute_error
 
 from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("regression")
 @tasks("regression")
-def MeanAbsoluteError(model, dataset, **kwargs):
+def MeanAbsoluteError(model: VMModel, dataset: VMDataset, **kwargs) -> float:
     """Calculates the mean absolute error for a regression model."""
     return _mean_absolute_error(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/regression/MeanAbsolutePercentageError.py

@@ -5,11 +5,12 @@
 import numpy as np
 
 from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("regression")
 @tasks("regression")
-def MeanAbsolutePercentageError(model, dataset):
+def MeanAbsolutePercentageError(model: VMModel, dataset: VMDataset) -> float:
     """Calculates the mean absolute percentage error for a regression model."""
     y_true = dataset.y
     y_pred = dataset.y_pred(model)

validmind/unit_metrics/regression/MeanBiasDeviation.py

@@ -5,10 +5,11 @@
 import numpy as np
 
 from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("regression")
 @tasks("regression")
-def MeanBiasDeviation(model, dataset):
+def MeanBiasDeviation(model: VMModel, dataset: VMDataset) -> float:
     """Calculates the mean bias deviation for a regression model."""
     return np.mean(dataset.y - dataset.y_pred(model))

validmind/unit_metrics/regression/MeanSquaredError.py

@@ -5,10 +5,11 @@
 from sklearn.metrics import mean_squared_error
 
 from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("regression")
 @tasks("regression")
-def MeanSquaredError(model, dataset, **kwargs):
+def MeanSquaredError(model: VMModel, dataset: VMDataset, **kwargs) -> float:
     """Calculates the mean squared error for a regression model."""
     return mean_squared_error(dataset.y, dataset.y_pred(model), **kwargs)

validmind/unit_metrics/regression/QuantileLoss.py

@@ -9,7 +9,7 @@ from validmind import tags, tasks
 
 @tags("regression")
 @tasks("regression")
-def QuantileLoss(model, dataset, quantile=0.5):
+def QuantileLoss(model, dataset, quantile=0.5) -> float:
     """Calculates the quantile loss for a regression model."""
     error = dataset.y - dataset.y_pred(model)
 

validmind/unit_metrics/regression/RSquaredScore.py

@@ -5,10 +5,11 @@
 from sklearn.metrics import r2_score
 
 from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("regression")
 @tasks("regression")
-def RSquaredScore(model, dataset):
+def RSquaredScore(model: VMModel, dataset: VMDataset) -> float:
     """Calculates the R-squared score for a regression model."""
     return r2_score(dataset.y, dataset.y_pred(model))

validmind/unit_metrics/regression/RootMeanSquaredError.py

@@ -6,11 +6,12 @@ import numpy as np
 from sklearn.metrics import mean_squared_error
 
 from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("regression")
 @tasks("regression")
-def RootMeanSquaredError(model, dataset, **kwargs):
+def RootMeanSquaredError(model: VMModel, dataset: VMDataset, **kwargs) -> float:
     """Calculates the root mean squared error for a regression model."""
     return np.sqrt(
         mean_squared_error(