validmind 2.8.29__py3-none-any.whl → 2.10.0rc1__py3-none-any.whl

validmind/unit_metrics/classification/individual/ClassBalance.py

@@ -0,0 +1,65 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def ClassBalance(model: VMModel, dataset: VMDataset, **kwargs) -> List[float]:
+    """Calculates the class balance score per row for a classification model.
+
+    For each prediction, this returns how balanced the predicted class is in the
+    training distribution. Lower scores indicate predictions on rare classes,
+    higher scores indicate predictions on common classes. This helps understand
+    if model errors are more likely on imbalanced classes.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predictions
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row class balance scores as a list of float values
+
+    Note:
+        Scores range from 0 to 0.5, where 0.5 indicates perfectly balanced classes
+        and lower values indicate more imbalanced classes.
+    """
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Convert to numpy arrays
+    y_true = np.asarray(y_true)
+    y_pred = np.asarray(y_pred)
+
+    # Calculate class frequencies in the true labels (proxy for training distribution)
+    unique_classes, class_counts = np.unique(y_true, return_counts=True)
+    class_frequencies = class_counts / len(y_true)
+
+    # Create a mapping from class to frequency
+    class_to_freq = dict(zip(unique_classes, class_frequencies))
+
+    # Calculate balance score for each prediction
+    balance_scores = []
+
+    for pred in y_pred:
+        if pred in class_to_freq:
+            freq = class_to_freq[pred]
+            # Balance score: how close to 0.5 (perfectly balanced) the frequency is
+            # Score = 0.5 - |freq - 0.5| = min(freq, 1-freq)
+            balance_score = min(freq, 1 - freq)
+        else:
+            # Predicted class not seen in true labels (very rare)
+            balance_score = 0.0
+
+        balance_scores.append(balance_score)
+
+    # Return as a list of floats
+    return balance_scores

validmind/unit_metrics/classification/individual/Confidence.py

@@ -0,0 +1,52 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def Confidence(model: VMModel, dataset: VMDataset, **kwargs) -> List[float]:
+    """Calculates the prediction confidence per row for a classification model.
+
+    For binary classification, confidence is calculated as the maximum probability
+    across classes, or alternatively as the distance from the decision boundary (0.5).
+    Higher values indicate more confident predictions.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predicted probabilities
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row confidence scores as a list of float values
+
+    Raises:
+        ValueError: If probability column is not found for the model
+    """
+    # Try to get probabilities, fall back to predictions if not available
+    try:
+        y_prob = dataset.y_prob(model)
+        # For binary classification, use max probability approach
+        if y_prob.ndim > 1 and y_prob.shape[1] > 1:
+            # Multi-class: confidence is the maximum probability
+            confidence = np.max(y_prob, axis=1)
+        else:
+            # Binary classification: confidence based on distance from 0.5
+            y_prob = np.asarray(y_prob, dtype=float)
+            confidence = np.abs(y_prob - 0.5) + 0.5
+    except ValueError:
+        # Fall back to binary correctness if probabilities not available
+        y_true = dataset.y
+        y_pred = dataset.y_pred(model)
+        # If no probabilities, confidence is 1.0 for correct, 0.0 for incorrect
+        confidence = (y_true == y_pred).astype(float)
+
+    # Return as a list of floats
+    return confidence.tolist()

validmind/unit_metrics/classification/individual/Correctness.py

@@ -0,0 +1,41 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def Correctness(model: VMModel, dataset: VMDataset, **kwargs) -> List[int]:
+    """Calculates the correctness per row for a classification model.
+
+    For classification tasks, this returns 1 for correctly classified rows
+    and 0 for incorrectly classified rows. This provides a binary indicator
+    of model performance for each individual prediction.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predictions
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[int]: Per-row correctness as a list of 1s and 0s
+    """
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Convert to numpy arrays
+    y_true = np.asarray(y_true)
+    y_pred = np.asarray(y_pred)
+
+    # For classification, check if predictions match true labels
+    correctness = (y_true == y_pred).astype(int)
+
+    # Return as a list of integers
+    return correctness.tolist()

validmind/unit_metrics/classification/individual/LogLoss.py

@@ -0,0 +1,61 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def LogLoss(
+    model: VMModel, dataset: VMDataset, eps: float = 1e-15, **kwargs
+) -> List[float]:
+    """Calculates the logarithmic loss per row for a classification model.
+
+    Log loss measures the performance of a classification model where the prediction
+    is a probability value between 0 and 1. The log loss increases as the predicted
+    probability diverges from the actual label.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predicted probabilities
+        eps: Small value to avoid log(0), defaults to 1e-15
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row log loss values as a list of float values
+
+    Raises:
+        ValueError: If probability column is not found for the model
+    """
+    y_true = dataset.y
+
+    # Try to get probabilities
+    try:
+        y_prob = dataset.y_prob(model)
+        # For binary classification, use the positive class probability
+        if y_prob.ndim > 1 and y_prob.shape[1] > 1:
+            y_prob = y_prob[:, 1]  # Use probability of positive class
+    except ValueError:
+        # Fall back to predictions if probabilities not available
+        # Convert predictions to "probabilities" (0.99 for correct class, 0.01 for wrong)
+        y_pred = dataset.y_pred(model)
+        y_prob = np.where(y_true == y_pred, 0.99, 0.01)
+
+    # Convert to numpy arrays and ensure same data type
+    y_true = np.asarray(y_true, dtype=float)
+    y_prob = np.asarray(y_prob, dtype=float)
+
+    # Clip probabilities to avoid log(0) and log(1)
+    y_prob = np.clip(y_prob, eps, 1 - eps)
+
+    # Calculate log loss per row: -[y*log(p) + (1-y)*log(1-p)]
+    log_loss_per_row = -(y_true * np.log(y_prob) + (1 - y_true) * np.log(1 - y_prob))
+
+    # Return as a list of floats
+    return log_loss_per_row.tolist()

validmind/unit_metrics/classification/individual/OutlierScore.py

@@ -0,0 +1,86 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+from sklearn.ensemble import IsolationForest
+from sklearn.preprocessing import StandardScaler
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def OutlierScore(
+    model: VMModel, dataset: VMDataset, contamination: float = 0.1, **kwargs
+) -> List[float]:
+    """Calculates the outlier score per row for a classification model.
+
+    Uses Isolation Forest to identify samples that deviate significantly from
+    the typical patterns in the feature space. Higher scores indicate more
+    anomalous/outlier-like samples. This can help identify out-of-distribution
+    samples or data points that might be harder to predict accurately.
+
+    Args:
+        model: The classification model to evaluate (unused but kept for consistency)
+        dataset: The dataset containing feature data
+        contamination: Expected proportion of outliers, defaults to 0.1
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row outlier scores as a list of float values
+
+    Note:
+        Scores are normalized to [0, 1] where higher values indicate more outlier-like samples
+    """
+    # Get feature data
+    X = dataset.x_df()
+
+    # Handle case where we have no features or only categorical features
+    if X.empty or X.shape[1] == 0:
+        # Return zero outlier scores if no features available
+        return [0.0] * len(dataset.y)
+
+    # Select only numeric features for outlier detection
+    numeric_features = dataset.feature_columns_numeric
+    if not numeric_features:
+        # If no numeric features, return zero outlier scores
+        return [0.0] * len(dataset.y)
+
+    X_numeric = X[numeric_features]
+
+    # Handle missing values by filling with median
+    X_filled = X_numeric.fillna(X_numeric.median())
+
+    # Standardize features for better outlier detection
+    scaler = StandardScaler()
+    X_scaled = scaler.fit_transform(X_filled)
+
+    # Fit Isolation Forest
+    isolation_forest = IsolationForest(
+        contamination=contamination, random_state=42, n_estimators=100
+    )
+
+    # Fit the model on the data
+    isolation_forest.fit(X_scaled)
+
+    # Get anomaly scores (negative values for outliers)
+    anomaly_scores = isolation_forest.decision_function(X_scaled)
+
+    # Convert to outlier scores (0 to 1, where 1 is most outlier-like)
+    # Normalize using min-max scaling
+    min_score = np.min(anomaly_scores)
+    max_score = np.max(anomaly_scores)
+
+    if max_score == min_score:
+        # All samples have same score, no outliers detected
+        outlier_scores = np.zeros_like(anomaly_scores)
+    else:
+        # Invert and normalize: higher values = more outlier-like
+        outlier_scores = (max_score - anomaly_scores) / (max_score - min_score)
+
+    # Return as a list of floats
+    return outlier_scores.tolist()

validmind/unit_metrics/classification/individual/ProbabilityError.py

@@ -0,0 +1,54 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def ProbabilityError(model: VMModel, dataset: VMDataset, **kwargs) -> List[float]:
+    """Calculates the probability error per row for a classification model.
+
+    For binary classification tasks, this computes the absolute difference between
+    the true class labels (0 or 1) and the predicted probabilities for each row.
+    This provides insight into how confident the model's predictions are and
+    how far off they are from the actual labels.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predicted probabilities
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row probability errors as a list of float values
+
+    Raises:
+        ValueError: If probability column is not found for the model
+    """
+    y_true = dataset.y
+
+    # Try to get probabilities, fall back to predictions if not available
+    try:
+        y_prob = dataset.y_prob(model)
+        # For binary classification, use the positive class probability
+        if y_prob.ndim > 1 and y_prob.shape[1] > 1:
+            y_prob = y_prob[:, 1]  # Use probability of positive class
+    except ValueError:
+        # Fall back to predictions if probabilities not available
+        y_prob = dataset.y_pred(model)
+
+    # Convert to numpy arrays and ensure same data type
+    y_true = np.asarray(y_true, dtype=float)
+    y_prob = np.asarray(y_prob, dtype=float)
+
+    # Compute absolute difference between true labels and predicted probabilities
+    probability_errors = np.abs(y_true - y_prob)
+
+    # Return as a list of floats
+    return probability_errors.tolist()

validmind/unit_metrics/classification/individual/Uncertainty.py

@@ -0,0 +1,60 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def Uncertainty(model: VMModel, dataset: VMDataset, **kwargs) -> List[float]:
+    """Calculates the prediction uncertainty per row for a classification model.
+
+    Uncertainty is measured using the entropy of the predicted probability distribution.
+    Higher entropy indicates higher uncertainty in the prediction. For binary
+    classification, maximum uncertainty occurs at probability 0.5.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predicted probabilities
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row uncertainty scores as a list of float values
+
+    Raises:
+        ValueError: If probability column is not found for the model
+    """
+    # Try to get probabilities
+    try:
+        y_prob = dataset.y_prob(model)
+
+        if y_prob.ndim > 1 and y_prob.shape[1] > 1:
+            # Multi-class: calculate entropy across all classes
+            # Clip to avoid log(0)
+            y_prob_clipped = np.clip(y_prob, 1e-15, 1 - 1e-15)
+            # Entropy: -sum(p * log(p))
+            uncertainty = -np.sum(y_prob_clipped * np.log(y_prob_clipped), axis=1)
+        else:
+            # Binary classification: calculate binary entropy
+            y_prob = np.asarray(y_prob, dtype=float)
+            # Clip to avoid log(0)
+            y_prob_clipped = np.clip(y_prob, 1e-15, 1 - 1e-15)
+            # Binary entropy: -[p*log(p) + (1-p)*log(1-p)]
+            uncertainty = -(
+                y_prob_clipped * np.log(y_prob_clipped)
+                + (1 - y_prob_clipped) * np.log(1 - y_prob_clipped)
+            )
+
+    except ValueError:
+        # If no probabilities available, assume zero uncertainty for hard predictions
+        n_samples = len(dataset.y)
+        uncertainty = np.zeros(n_samples)
+
+    # Return as a list of floats
+    return uncertainty.tolist()

validmind/vm_models/dataset/dataset.py

@@ -8,7 +8,7 @@ Dataset class wrapper
 
 import warnings
 from copy import deepcopy
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -458,6 +458,152 @@ class VMDataset(VMInput):
 
         return self.extra_columns.probability_column(model, column_name)
 
+    def assign_scores(
+        self,
+        model: VMModel,
+        metrics: Union[str, List[str]],
+        **kwargs: Dict[str, Any],
+    ) -> None:
+        """Assign computed unit metric scores to the dataset as new columns.
+
+        This method computes unit metrics for the given model and dataset, then adds
+        the computed scores as new columns to the dataset using the naming convention:
+        {model.input_id}_{metric_name}
+
+        Args:
+            model (VMModel): The model used to compute the scores.
+            metrics (Union[str, List[str]]): Single metric ID or list of metric IDs.
+                Can be either:
+                - Short name (e.g., "F1", "Precision")
+                - Full metric ID (e.g., "validmind.unit_metrics.classification.F1")
+            **kwargs: Additional parameters passed to the unit metrics.
+
+        Examples:
+            # Single metric
+            dataset.assign_scores(model, "F1")
+
+            # Multiple metrics
+            dataset.assign_scores(model, ["F1", "Precision", "Recall"])
+
+            # With parameters
+            dataset.assign_scores(model, "ROC_AUC", average="weighted")
+
+        Raises:
+            ValueError: If the model input_id is None or if metric computation fails.
+            ImportError: If unit_metrics module cannot be imported.
+        """
+        if model.input_id is None:
+            raise ValueError("Model input_id must be set to use assign_scores")
+
+        # Import unit_metrics module
+        try:
+            from validmind.unit_metrics import run_metric
+        except ImportError as e:
+            raise ImportError(
+                f"Failed to import unit_metrics module: {e}. "
+                "Make sure validmind.unit_metrics is available."
+            ) from e
+
+        # Normalize metrics to a list
+        if isinstance(metrics, str):
+            metrics = [metrics]
+
+        # Process each metric
+        for metric in metrics:
+            # Normalize metric ID
+            metric_id = self._normalize_metric_id(metric)
+
+            # Extract metric name for column naming
+            metric_name = self._extract_metric_name(metric_id)
+
+            # Generate column name
+            column_name = f"{model.input_id}_{metric_name}"
+
+            try:
+                # Run the unit metric
+                result = run_metric(
+                    metric_id,
+                    inputs={
+                        "model": model,
+                        "dataset": self,
+                    },
+                    params=kwargs,
+                    show=False,  # Don't show widget output
+                )
+
+                # Extract the metric value
+                metric_value = result.metric
+
+                # Create column values (repeat the scalar value for all rows)
+                if np.isscalar(metric_value):
+                    column_values = np.full(len(self._df), metric_value)
+                else:
+                    if len(metric_value) != len(self._df):
+                        raise ValueError(
+                            f"Metric value length {len(metric_value)} does not match dataset length {len(self._df)}"
+                        )
+                    column_values = metric_value
+
+                # Add the column to the dataset
+                self.add_extra_column(column_name, column_values)
+
+                logger.info(f"Added metric column '{column_name}'")
+            except Exception as e:
+                logger.error(f"Failed to compute metric {metric_id}: {e}")
+                raise ValueError(f"Failed to compute metric {metric_id}: {e}") from e
+
+    def _normalize_metric_id(self, metric: str) -> str:
+        """Normalize metric identifier to full validmind unit metric ID.
+
+        Args:
+            metric (str): Metric identifier (short name or full ID)
+
+        Returns:
+            str: Full metric ID
+        """
+        # If already a full ID, return as-is
+        if metric.startswith("validmind.unit_metrics."):
+            return metric
+
+        # Try to find the metric by short name
+        try:
+            from validmind.unit_metrics import list_metrics
+
+            available_metrics = list_metrics()
+
+            # Look for exact match with short name
+            for metric_id in available_metrics:
+                if metric_id.endswith(f".{metric}"):
+                    return metric_id
+
+            # If no exact match found, raise error with suggestions
+            suggestions = [m for m in available_metrics if metric.lower() in m.lower()]
+            if suggestions:
+                raise ValueError(
+                    f"Metric '{metric}' not found. Did you mean one of: {suggestions[:5]}"
+                )
+            else:
+                raise ValueError(
+                    f"Metric '{metric}' not found. Available metrics: {available_metrics[:10]}..."
+                )
+
+        except ImportError as e:
+            raise ImportError(
+                f"Failed to import unit_metrics for metric lookup: {e}"
+            ) from e
+
+    def _extract_metric_name(self, metric_id: str) -> str:
+        """Extract the metric name from a full metric ID.
+
+        Args:
+            metric_id (str): Full metric ID
+
+        Returns:
+            str: Metric name
+        """
+        # Extract the last part after the final dot
+        return metric_id.split(".")[-1]
+
     def add_extra_column(self, column_name, column_values=None):
         """Adds an extra column to the dataset without modifying the dataset `features` and `target` columns.
 

validmind/vm_models/result/result.py

@@ -7,6 +7,7 @@ Result objects for test results
 """
 import asyncio
 import json
+import os
 from abc import abstractmethod
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Union
@@ -20,7 +21,7 @@ from ipywidgets import HTML, VBox
 from ... import api_client
 from ...ai.utils import DescriptionFuture
 from ...errors import InvalidParameterError
-from ...logging import get_logger
+from ...logging import get_logger, log_api_operation
 from ...utils import (
     HumanReadableEncoder,
     NumpyEncoder,
@@ -177,7 +178,7 @@ class TestResult(Result):
     title: Optional[str] = None
     doc: Optional[str] = None
     description: Optional[Union[str, DescriptionFuture]] = None
-    metric: Optional[Union[int, float]] = None
+    metric: Optional[Union[int, float, List[Union[int, float]]]] = None
     tables: Optional[List[ResultTable]] = None
     raw_data: Optional[RawData] = None
     figures: Optional[List[Figure]] = None
@@ -464,8 +465,10 @@ class TestResult(Result):
             )
         )
 
-        if self.metric is not None:
-            # metrics are logged as separate entities
+        # Only log unit metrics when the metric is a scalar value.
+        # Some tests may assign a list/array of per-row metrics to `self.metric`.
+        # Those should not be sent to the unit-metric endpoint which expects scalars.
+        if self.metric is not None and not hasattr(self.metric, "__len__"):
             tasks.append(
                 api_client.alog_metric(
                     key=self.result_id,
@@ -476,9 +479,30 @@ class TestResult(Result):
             )
 
         if self.figures:
-            tasks.extend(
-                [api_client.alog_figure(figure) for figure in (self.figures or [])]
+            batch_size = min(
+                len(self.figures), int(os.getenv("VM_FIGURE_MAX_BATCH_SIZE", 20))
             )
+            figure_batches = [
+                self.figures[i : i + batch_size]
+                for i in range(0, len(self.figures), batch_size)
+            ]
+
+            async def upload_figures_in_batches():
+                for batch in figure_batches:
+
+                    @log_api_operation(
+                        operation_name=f"Uploading batch of {len(batch)} figures"
+                    )
+                    async def process_batch():
+                        batch_tasks = [
+                            api_client.alog_figure(figure) for figure in batch
+                        ]
+                        return await asyncio.gather(*batch_tasks)
+
+                    await process_batch()
+
+            tasks.append(upload_figures_in_batches())
+
         if self.description:
             revision_name = (
                 AI_REVISION_NAME