validmind 2.8.28__py3-none-any.whl → 2.9.1__py3-none-any.whl

validmind/vm_models/dataset/dataset.py

@@ -8,7 +8,7 @@ Dataset class wrapper
 
 import warnings
 from copy import deepcopy
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -258,69 +258,91 @@ class VMDataset(VMInput):
                 f"Options {kwargs} are not supported for this input"
             )
 
-    def assign_predictions(
-        self,
-        model: VMModel,
-        prediction_column: Optional[str] = None,
-        prediction_values: Optional[List[Any]] = None,
-        probability_column: Optional[str] = None,
-        probability_values: Optional[List[float]] = None,
-        prediction_probabilities: Optional[
-            List[float]
-        ] = None,  # DEPRECATED: use probability_values
-        **kwargs: Dict[str, Any],
-    ) -> None:
-        """Assign predictions and probabilities to the dataset.
-
-        Args:
-            model (VMModel): The model used to generate the predictions.
-            prediction_column (Optional[str]): The name of the column containing the predictions.
-            prediction_values (Optional[List[Any]]): The values of the predictions.
-            probability_column (Optional[str]): The name of the column containing the probabilities.
-            probability_values (Optional[List[float]]): The values of the probabilities.
-            prediction_probabilities (Optional[List[float]]): DEPRECATED: The values of the probabilities.
-            **kwargs: Additional keyword arguments that will get passed through to the model's `predict` method.
-        """
+    def _handle_deprecated_parameters(
+        self, prediction_probabilities, probability_values
+    ):
+        """Handle deprecated parameters and return the correct probability values."""
         if prediction_probabilities is not None:
             warnings.warn(
                 "The `prediction_probabilities` argument is deprecated. Use `probability_values` instead.",
                 DeprecationWarning,
             )
-            probability_values = prediction_probabilities
-
-        self._validate_assign_predictions(
-            model,
-            prediction_column,
-            prediction_values,
-            probability_column,
-            probability_values,
-        )
+            return prediction_probabilities
+        return probability_values
 
+    def _check_existing_predictions(self, model):
+        """Check for existing predictions and probabilities, warn if overwriting."""
         if self.prediction_column(model):
             logger.warning("Model predictions already assigned... Overwriting.")
 
         if self.probability_column(model):
             logger.warning("Model probabilities already assigned... Overwriting.")
 
-        # if the user passes a column name, we assume it has precomputed predictions
+    def _get_precomputed_values(self, prediction_column, probability_column):
+        """Get precomputed prediction and probability values from existing columns."""
+        prediction_values = None
+        probability_values = None
+
         if prediction_column:
             prediction_values = self._df[prediction_column].values
 
             if probability_column:
                 probability_values = self._df[probability_column].values
 
+        return prediction_values, probability_values
+
+    def _compute_predictions_if_needed(self, model, prediction_values, **kwargs):
+        """Compute predictions if not provided."""
         if prediction_values is None:
             X = self.df if isinstance(model, (FunctionModel, PipelineModel)) else self.x
-            probability_values, prediction_values = compute_predictions(
-                model, X, **kwargs
+            return compute_predictions(model, X, **kwargs)
+        return None, prediction_values
+
+    def _handle_dictionary_predictions(self, model, prediction_values):
+        """Handle dictionary predictions by converting to separate columns."""
+        if (
+            prediction_values is not None
+            and len(prediction_values) > 0
+            and isinstance(prediction_values[0], dict)
+        ):
+            df_prediction_values = pd.DataFrame.from_dict(
+                prediction_values, orient="columns"
             )
 
-        prediction_column = prediction_column or f"{model.input_id}_prediction"
+            for column_name in df_prediction_values.columns.tolist():
+                values = df_prediction_values[column_name].values
+
+                if column_name == "prediction":
+                    prediction_column = f"{model.input_id}_prediction"
+                    self._add_column(prediction_column, values)
+                    self.prediction_column(model, prediction_column)
+                else:
+                    self._add_column(f"{model.input_id}_{column_name}", values)
+
+            return (
+                True,
+                None,
+            )  # Return True to indicate dictionary handled, None for prediction_column
+        return False, None
+
+    def _add_prediction_columns(
+        self,
+        model,
+        prediction_column,
+        prediction_values,
+        probability_column,
+        probability_values,
+    ):
+        """Add prediction and probability columns to the dataset."""
+        if prediction_column is None:
+            prediction_column = f"{model.input_id}_prediction"
+
         self._add_column(prediction_column, prediction_values)
         self.prediction_column(model, prediction_column)
 
         if probability_values is not None:
-            probability_column = probability_column or f"{model.input_id}_probabilities"
+            if probability_column is None:
+                probability_column = f"{model.input_id}_probabilities"
             self._add_column(probability_column, probability_values)
             self.probability_column(model, probability_column)
         else:
@@ -329,6 +351,91 @@ class VMDataset(VMInput):
                 "Not adding probability column to the dataset."
             )
 
+    def assign_predictions(
+        self,
+        model: VMModel,
+        prediction_column: Optional[str] = None,
+        prediction_values: Optional[Any] = None,
+        probability_column: Optional[str] = None,
+        probability_values: Optional[Any] = None,
+        prediction_probabilities: Optional[
+            Any
+        ] = None,  # DEPRECATED: use probability_values
+        **kwargs: Dict[str, Any],
+    ) -> None:
+        """Assign predictions and probabilities to the dataset.
+
+        Args:
+            model (VMModel): The model used to generate the predictions.
+            prediction_column (Optional[str]): The name of the column containing the predictions.
+            prediction_values (Optional[Any]): The values of the predictions. Can be array-like (list, numpy array, pandas Series, etc.).
+            probability_column (Optional[str]): The name of the column containing the probabilities.
+            probability_values (Optional[Any]): The values of the probabilities. Can be array-like (list, numpy array, pandas Series, etc.).
+            prediction_probabilities (Optional[Any]): DEPRECATED: The values of the probabilities. Use probability_values instead.
+            **kwargs: Additional keyword arguments that will get passed through to the model's `predict` method.
+        """
+        # Handle deprecated parameters
+        probability_values = self._handle_deprecated_parameters(
+            prediction_probabilities, probability_values
+        )
+
+        # Convert pandas Series to numpy array for prediction_values
+        if (
+            hasattr(prediction_values, "values")
+            and hasattr(prediction_values, "index")
+            and hasattr(prediction_values, "dtype")
+        ):
+            prediction_values = prediction_values.values
+
+        # Convert pandas Series to numpy array for probability_values
+        if (
+            hasattr(probability_values, "values")
+            and hasattr(probability_values, "index")
+            and hasattr(probability_values, "dtype")
+        ):
+            probability_values = probability_values.values
+
+        # Validate input parameters
+        self._validate_assign_predictions(
+            model,
+            prediction_column,
+            prediction_values,
+            probability_column,
+            probability_values,
+        )
+
+        # Check for existing predictions and warn if overwriting
+        self._check_existing_predictions(model)
+
+        # Get precomputed values if column names are provided
+        if prediction_column or probability_column:
+            prediction_values, prob_values_from_column = self._get_precomputed_values(
+                prediction_column, probability_column
+            )
+            if prob_values_from_column is not None:
+                probability_values = prob_values_from_column
+
+        # Compute predictions if not provided
+        if prediction_values is None:
+            probability_values, prediction_values = self._compute_predictions_if_needed(
+                model, prediction_values, **kwargs
+            )
+
+        # Handle dictionary predictions
+        is_dict_handled, _ = self._handle_dictionary_predictions(
+            model, prediction_values
+        )
+
+        # Add prediction and probability columns (skip if dictionary was handled)
+        if not is_dict_handled:
+            self._add_prediction_columns(
+                model,
+                prediction_column,
+                prediction_values,
+                probability_column,
+                probability_values,
+            )
+
     def prediction_column(self, model: VMModel, column_name: str = None) -> str:
         """Get or set the prediction column for a model."""
         if column_name and column_name not in self.columns:
@@ -351,6 +458,152 @@ class VMDataset(VMInput):
 
         return self.extra_columns.probability_column(model, column_name)
 
+    def assign_scores(
+        self,
+        model: VMModel,
+        metrics: Union[str, List[str]],
+        **kwargs: Dict[str, Any],
+    ) -> None:
+        """Assign computed unit metric scores to the dataset as new columns.
+
+        This method computes unit metrics for the given model and dataset, then adds
+        the computed scores as new columns to the dataset using the naming convention:
+        {model.input_id}_{metric_name}
+
+        Args:
+            model (VMModel): The model used to compute the scores.
+            metrics (Union[str, List[str]]): Single metric ID or list of metric IDs.
+                Can be either:
+                - Short name (e.g., "F1", "Precision")
+                - Full metric ID (e.g., "validmind.unit_metrics.classification.F1")
+            **kwargs: Additional parameters passed to the unit metrics.
+
+        Examples:
+            # Single metric
+            dataset.assign_scores(model, "F1")
+
+            # Multiple metrics
+            dataset.assign_scores(model, ["F1", "Precision", "Recall"])
+
+            # With parameters
+            dataset.assign_scores(model, "ROC_AUC", average="weighted")
+
+        Raises:
+            ValueError: If the model input_id is None or if metric computation fails.
+            ImportError: If unit_metrics module cannot be imported.
+        """
+        if model.input_id is None:
+            raise ValueError("Model input_id must be set to use assign_scores")
+
+        # Import unit_metrics module
+        try:
+            from validmind.unit_metrics import run_metric
+        except ImportError as e:
+            raise ImportError(
+                f"Failed to import unit_metrics module: {e}. "
+                "Make sure validmind.unit_metrics is available."
+            ) from e
+
+        # Normalize metrics to a list
+        if isinstance(metrics, str):
+            metrics = [metrics]
+
+        # Process each metric
+        for metric in metrics:
+            # Normalize metric ID
+            metric_id = self._normalize_metric_id(metric)
+
+            # Extract metric name for column naming
+            metric_name = self._extract_metric_name(metric_id)
+
+            # Generate column name
+            column_name = f"{model.input_id}_{metric_name}"
+
+            try:
+                # Run the unit metric
+                result = run_metric(
+                    metric_id,
+                    inputs={
+                        "model": model,
+                        "dataset": self,
+                    },
+                    params=kwargs,
+                    show=False,  # Don't show widget output
+                )
+
+                # Extract the metric value
+                metric_value = result.metric
+
+                # Create column values (repeat the scalar value for all rows)
+                if np.isscalar(metric_value):
+                    column_values = np.full(len(self._df), metric_value)
+                else:
+                    if len(metric_value) != len(self._df):
+                        raise ValueError(
+                            f"Metric value length {len(metric_value)} does not match dataset length {len(self._df)}"
+                        )
+                    column_values = metric_value
+
+                # Add the column to the dataset
+                self.add_extra_column(column_name, column_values)
+
+                logger.info(f"Added metric column '{column_name}'")
+            except Exception as e:
+                logger.error(f"Failed to compute metric {metric_id}: {e}")
+                raise ValueError(f"Failed to compute metric {metric_id}: {e}") from e
+
+    def _normalize_metric_id(self, metric: str) -> str:
+        """Normalize metric identifier to full validmind unit metric ID.
+
+        Args:
+            metric (str): Metric identifier (short name or full ID)
+
+        Returns:
+            str: Full metric ID
+        """
+        # If already a full ID, return as-is
+        if metric.startswith("validmind.unit_metrics."):
+            return metric
+
+        # Try to find the metric by short name
+        try:
+            from validmind.unit_metrics import list_metrics
+
+            available_metrics = list_metrics()
+
+            # Look for exact match with short name
+            for metric_id in available_metrics:
+                if metric_id.endswith(f".{metric}"):
+                    return metric_id
+
+            # If no exact match found, raise error with suggestions
+            suggestions = [m for m in available_metrics if metric.lower() in m.lower()]
+            if suggestions:
+                raise ValueError(
+                    f"Metric '{metric}' not found. Did you mean one of: {suggestions[:5]}"
+                )
+            else:
+                raise ValueError(
+                    f"Metric '{metric}' not found. Available metrics: {available_metrics[:10]}..."
+                )
+
+        except ImportError as e:
+            raise ImportError(
+                f"Failed to import unit_metrics for metric lookup: {e}"
+            ) from e
+
+    def _extract_metric_name(self, metric_id: str) -> str:
+        """Extract the metric name from a full metric ID.
+
+        Args:
+            metric_id (str): Full metric ID
+
+        Returns:
+            str: Metric name
+        """
+        # Extract the last part after the final dot
+        return metric_id.split(".")[-1]
+
     def add_extra_column(self, column_name, column_values=None):
         """Adds an extra column to the dataset without modifying the dataset `features` and `target` columns.
 

validmind/vm_models/result/result.py

@@ -7,6 +7,7 @@ Result objects for test results
 """
 import asyncio
 import json
+import os
 from abc import abstractmethod
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Union
@@ -20,7 +21,7 @@ from ipywidgets import HTML, VBox
 from ... import api_client
 from ...ai.utils import DescriptionFuture
 from ...errors import InvalidParameterError
-from ...logging import get_logger
+from ...logging import get_logger, log_api_operation
 from ...utils import (
     HumanReadableEncoder,
     NumpyEncoder,
@@ -177,7 +178,7 @@ class TestResult(Result):
     title: Optional[str] = None
     doc: Optional[str] = None
     description: Optional[Union[str, DescriptionFuture]] = None
-    metric: Optional[Union[int, float]] = None
+    metric: Optional[Union[int, float, List[Union[int, float]]]] = None
     tables: Optional[List[ResultTable]] = None
     raw_data: Optional[RawData] = None
     figures: Optional[List[Figure]] = None
@@ -476,9 +477,30 @@ class TestResult(Result):
             )
 
         if self.figures:
-            tasks.extend(
-                [api_client.alog_figure(figure) for figure in (self.figures or [])]
+            batch_size = min(
+                len(self.figures), int(os.getenv("VM_FIGURE_MAX_BATCH_SIZE", 20))
             )
+            figure_batches = [
+                self.figures[i : i + batch_size]
+                for i in range(0, len(self.figures), batch_size)
+            ]
+
+            async def upload_figures_in_batches():
+                for batch in figure_batches:
+
+                    @log_api_operation(
+                        operation_name=f"Uploading batch of {len(batch)} figures"
+                    )
+                    async def process_batch():
+                        batch_tasks = [
+                            api_client.alog_figure(figure) for figure in batch
+                        ]
+                        return await asyncio.gather(*batch_tasks)
+
+                    await process_batch()
+
+            tasks.append(upload_figures_in_batches())
+
         if self.description:
             revision_name = (
                 AI_REVISION_NAME

{validmind-2.8.28.dist-info → validmind-2.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: validmind
-Version: 2.8.28
+Version: 2.9.1
 Summary: ValidMind Library
 License: Commercial License
 Author: Andres Rodriguez
@@ -24,7 +24,7 @@ Requires-Dist: datasets (>=2.10.0,<3.0.0)
 Requires-Dist: evaluate
 Requires-Dist: h11 (>=0.16.0)
 Requires-Dist: ipywidgets
-Requires-Dist: kaleido (>=0.2.1,!=0.2.1.post1)
+Requires-Dist: kaleido (>=0.2.1,!=0.2.1.post1,<1.0.0)
 Requires-Dist: langchain-openai (>=0.1.8) ; extra == "all" or extra == "llm"
 Requires-Dist: langdetect
 Requires-Dist: llvmlite ; python_version >= "3.8" and python_full_version <= "3.11.0"