validmind 2.5.25__py3-none-any.whl → 2.6.8__py3-none-any.whl

validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py

@@ -3,22 +3,122 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
-from dataclasses import dataclass
+from warnings import filters as _warnings_filters
 
 import matplotlib.pyplot as plt
 import numpy as np
 import shap
 
+from validmind import tags, tasks
 from validmind.errors import UnsupportedModelForSHAPError
 from validmind.logging import get_logger
 from validmind.models import CatBoostModel, SKlearnModel, StatsModelsModel
-from validmind.vm_models import Figure, Metric
+from validmind.vm_models import VMDataset, VMModel
 
 logger = get_logger(__name__)
 
 
-@dataclass
-class SHAPGlobalImportance(Metric):
+def select_shap_values(shap_values, class_of_interest):
+    """Selects SHAP values for binary or multiclass classification.
+
+    For regression models, returns the SHAP values directly as there are no classes.
+
+    Args:
+        shap_values: The SHAP values returned by the SHAP explainer. For multiclass
+            classification, this will be a list where each element corresponds to a class.
+            For regression, this will be a single array of SHAP values.
+        class_of_interest: The class index for which to retrieve SHAP values. If None
+            (default), the function will assume binary classification and use class 1
+            by default.
+
+    Returns:
+        The SHAP values for the specified class (classification) or for the regression
+        output.
+
+    Raises:
+        ValueError: If class_of_interest is specified and is out of bounds for the
+            number of classes.
+    """
+    if not isinstance(shap_values, list):
+        # For regression, return the SHAP values as they are
+        # TODO: shap_values is always an array of all predictions, how is the if above supposed to work?
+        # logger.info("Returning SHAP values as-is.")
+        return shap_values
+
+    num_classes = len(shap_values)
+
+    # Default to class 1 for binary classification where no class is specified
+    if num_classes == 2 and class_of_interest is None:
+        logger.debug("Using SHAP values for class 1 (positive class).")
+        return shap_values[1]
+
+    # Otherwise, use the specified class_of_interest
+    if (
+        class_of_interest is None
+        or class_of_interest < 0
+        or class_of_interest >= num_classes
+    ):
+        raise ValueError(
+            f"Invalid class_of_interest: {class_of_interest}. Must be between 0 and {num_classes - 1}."
+        )
+
+    logger.debug(f"Using SHAP values for class {class_of_interest}.")
+    return shap_values[class_of_interest]
+
+
+def generate_shap_plot(type_, shap_values, x_test):
+    """Plots two types of SHAP global importance (SHAP).
+
+    Args:
+        type_: The type of SHAP plot to generate. Must be "mean" or "summary".
+        shap_values: The SHAP values to plot.
+        x_test: The test data used to generate the SHAP values.
+
+    Returns:
+        The generated plot.
+    """
+    ax = plt.axes()
+    ax.set_facecolor("white")
+
+    if type_ == "mean":
+        # Calculate the mean absolute SHAP value for each feature
+        mean_abs_shap = np.abs(shap_values).mean(axis=0)
+        # Find the maximum mean absolute SHAP value
+        max_shap_value = np.max(mean_abs_shap)
+        # Normalize all SHAP values based on the top feature
+        shap_values = shap_values / max_shap_value * 100
+
+        shap.summary_plot(shap_values, x_test, show=False, plot_type="bar")
+
+        # Customize the plot using matplotlib
+        plt.xlabel("Normalized SHAP Value (Percentage)", fontsize=13)
+        plt.ylabel("Features", fontsize=13)
+        plt.title("Normalized Feature Importance", fontsize=13)
+    else:
+        shap.summary_plot(shap_values, x_test, show=False)
+
+    fig = plt.gcf()
+
+    plt.close()
+
+    return fig
+
+
+@tags(
+    "sklearn",
+    "binary_classification",
+    "multiclass_classification",
+    "feature_importance",
+    "visualization",
+)
+@tasks("classification", "text_classification")
+def SHAPGlobalImportance(
+    model: VMModel,
+    dataset: VMDataset,
+    kernel_explainer_samples: int = 10,
+    tree_or_linear_explainer_samples: int = 200,
+    class_of_interest: int = None,
+):
     """
     Evaluates and visualizes global feature importance using SHAP values for model explanation and risk identification.
 
@@ -44,7 +144,6 @@ class SHAPGlobalImportance(Metric):
     represents a Shapley value for a certain feature in a specific case. The vertical axis is denoted by the feature
     whereas the horizontal one corresponds to the Shapley value. A color gradient indicates the value of the feature,
     gradually changing from low to high. Features are systematically organized in accordance with their importance.
-    These plots are generated by the function `_generate_shap_plot()`.
 
     ### Signs of High Risk
 
@@ -64,213 +163,70 @@ class SHAPGlobalImportance(Metric):
     - High-dimensional data can convolute interpretations.
     - Associating importance with tangible real-world impact still involves a certain degree of subjectivity.
     """
-
-    name = "shap"
-    required_inputs = ["model", "dataset"]
-    tasks = ["classification", "text_classification"]
-    tags = [
-        "sklearn",
-        "binary_classification",
-        "multiclass_classification",
-        "feature_importance",
-        "visualization",
-    ]
-    default_params = {
-        "kernel_explainer_samples": 10,
-        "tree_or_linear_explainer_samples": 200,
-        "class_of_interest": None,
-    }
-
-    def _generate_shap_plot(self, type_, shap_values, x_test):
-        """
-        Plots two types of SHAP global importance (SHAP).
-        :params type: mean, summary
-        :params shap_values: a matrix
-        :params x_test:
-        """
-        plt.close("all")
-
-        # preserve styles
-        # mpl.rcParams["grid.color"] = "#CCC"
-        ax = plt.axes()
-        ax.set_facecolor("white")
-
-        summary_plot_extra_args = {}
-        if type_ == "mean":
-            # Calculate the mean absolute SHAP value for each feature
-            mean_abs_shap = np.abs(shap_values).mean(axis=0)
-
-            # Find the maximum mean absolute SHAP value
-            max_shap_value = np.max(mean_abs_shap)
-
-            # Normalize all SHAP values based on the top feature
-            shap_values = (
-                shap_values / max_shap_value * 100
-            )  # scaling factor to make the top feature 100%
-            summary_plot_extra_args = {"plot_type": "bar"}
-
-            shap.summary_plot(
-                shap_values, x_test, show=False, **summary_plot_extra_args
-            )
-
-            # Customize the plot using matplotlib
-            plt.xlabel("Normalized SHAP Value (Percentage)", fontsize=13)
-            plt.ylabel("Features", fontsize=13)
-            plt.title("Normalized Feature Importance", fontsize=13)
-        else:
-            shap.summary_plot(
-                shap_values, x_test, show=False, **summary_plot_extra_args
-            )
-
-        figure = plt.gcf()
-        # avoid displaying on notebooks and clears the canvas for the next plot
-        plt.close()
-
-        return Figure(
-            for_object=self,
-            figure=figure,
-            key=f"shap:{type_}",
-            metadata={"type": type_},
+    if not isinstance(model, SKlearnModel) or isinstance(
+        model, (CatBoostModel, StatsModelsModel)
+    ):
+        raise UnsupportedModelForSHAPError(
+            f"Model {model.class_} is not supported for SHAP importance."
         )
 
-    def run(self):
-        if not isinstance(self.inputs.model, SKlearnModel) or isinstance(
-            self.inputs.model, (CatBoostModel, StatsModelsModel)
-        ):
-            logger.info(f"Skiping SHAP for {self.inputs.model.library} models")
-            return
-
-        trained_model = self.inputs.model.model
-        model_class = self.inputs.model.class_
-
-        # the shap library generates a bunch of annoying warnings that we don't care about
-        warnings.filterwarnings("ignore", category=UserWarning)
-
-        # Any tree based model can go here
-        if (
-            model_class == "XGBClassifier"
-            or model_class == "RandomForestClassifier"
-            or model_class == "CatBoostClassifier"
-            or model_class == "DecisionTreeClassifier"
-            or model_class == "RandomForestRegressor"
-            or model_class == "GradientBoostingRegressor"
-        ):
-            explainer = shap.TreeExplainer(trained_model)
-        elif (
-            model_class == "LogisticRegression"
-            or model_class == "XGBRegressor"
-            or model_class == "LinearRegression"
-            or model_class == "LinearSVC"
-        ):
-            explainer = shap.LinearExplainer(trained_model, self.inputs.dataset.x)
-        elif model_class == "SVC":
-            # KernelExplainer is slow so we use shap.sample to speed it up
-            explainer = shap.KernelExplainer(
-                trained_model.predict,
-                shap.sample(
-                    self.inputs.dataset.x,
-                    self.params["kernel_explainer_samples"],
-                ),
-            )
-        else:
-            model_class = "<ExternalModel>" if model_class is None else model_class
-            raise UnsupportedModelForSHAPError(
-                f"Model {model_class} not supported for SHAP importance."
-            )
-
+    model_class = model.class_
+
+    # the shap library generates a bunch of annoying warnings that we don't care about
+    warnings.filterwarnings("ignore", category=UserWarning)
+
+    if (
+        model_class == "XGBClassifier"
+        or model_class == "RandomForestClassifier"
+        or model_class == "CatBoostClassifier"
+        or model_class == "DecisionTreeClassifier"
+        or model_class == "RandomForestRegressor"
+        or model_class == "GradientBoostingRegressor"
+    ):
+        explainer = shap.TreeExplainer(model.model)
+    elif (
+        model_class == "LogisticRegression"
+        or model_class == "XGBRegressor"
+        or model_class == "LinearRegression"
+        or model_class == "LinearSVC"
+    ):
+        explainer = shap.LinearExplainer(model.model, dataset.x)
+    elif model_class == "SVC":
         # KernelExplainer is slow so we use shap.sample to speed it up
-        if isinstance(explainer, shap.KernelExplainer):
-            shap_sample = shap.sample(
-                self.inputs.dataset.x_df(),
-                self.params["kernel_explainer_samples"],
-            )
-        else:
-            shap_sample = self.inputs.dataset.x_df().sample(
-                min(
-                    self.params["tree_or_linear_explainer_samples"],
-                    self.inputs.dataset.x_df().shape[0],
-                )
-            )
-
-        shap_values = explainer.shap_values(shap_sample)
-
-        # Select the SHAP values for the specified class (classification) or for the regression output.
-        class_of_interest = self.params["class_of_interest"]
-        shap_values = _select_shap_values(shap_values, class_of_interest)
-
-        figures = [
-            self._generate_shap_plot("mean", shap_values, shap_sample),
-            self._generate_shap_plot("summary", shap_values, shap_sample),
-        ]
-
-        # restore warnings
-        warnings.filterwarnings("default", category=UserWarning)
-
-        return self.cache_results(figures=figures)
-
-    def test(self):
-        """Unit Test for SHAP Global Importance Metric"""
-        # Verify that the result object is not None
-        assert self.result is not None
-
-        # Verify that there are exactly two figures in the figures list
-        assert len(self.result.figures) == 2
-
-        # Verify that each figure is an instance of Figure and has the correct metadata type
-        for fig_num, type_ in enumerate(["mean", "summary"], start=1):
-            assert isinstance(self.result.figures[fig_num - 1], Figure)
-            assert self.result.figures[fig_num - 1].metadata["type"] == type_
-
-
-def _select_shap_values(shap_values, class_of_interest=None):
-    """
-    Selects SHAP values for binary or multiclass classification. For regression models,
-    returns the SHAP values directly as there are no classes.
+        explainer = shap.KernelExplainer(
+            model.model.predict,
+            shap.sample(
+                dataset.x,
+                kernel_explainer_samples,
+            ),
+        )
+    else:
+        model_class = "<ExternalModel>" if model_class is None else model_class
+        raise UnsupportedModelForSHAPError(
+            f"Model {model_class} not supported for SHAP importance."
+        )
 
-    Parameters:
-    -----------
-    shap_values : list or numpy.ndarray
-        The SHAP values returned by the SHAP explainer. For multiclass classification,
-        this will be a list where each element corresponds to a class. For regression,
-        this will be a single array of SHAP values.
+    # KernelExplainer is slow so we use shap.sample to speed it up
+    if isinstance(explainer, shap.KernelExplainer):
+        shap_sample = shap.sample(
+            dataset.x,
+            kernel_explainer_samples,
+        )
+    else:
+        shap_sample = dataset.x_df().sample(
+            min(
+                tree_or_linear_explainer_samples,
+                dataset.x_df().shape[0],
+            )
+        )
 
-    class_of_interest : int, optional
-        The class index for which to retrieve SHAP values. If None (default), the function
-        will assume binary classification and use class 1 by default.
+    shap_values = explainer.shap_values(shap_sample)
+    shap_values = select_shap_values(shap_values, class_of_interest)
 
-    Returns:
-    --------
-    numpy.ndarray
-        The SHAP values for the specified class (classification) or for the regression output.
+    # restore warnings
+    _warnings_filters.pop(0)
 
-    Raises:
-    -------
-    ValueError
-        If class_of_interest is specified and is out of bounds for the number of classes.
-    """
-    # Check if we are dealing with a multiclass classification
-    if isinstance(shap_values, list):
-        num_classes = len(shap_values)
-
-        # Default to class 1 for binary classification
-        if num_classes == 2 and class_of_interest is None:
-            logger.info(
-                "Binary classification detected: using SHAP values for class 1 (positive class)."
-            )
-            return shap_values[1]
-        else:
-            # Multiclass classification: use the specified class_of_interest
-            if class_of_interest is not None and 0 <= class_of_interest < num_classes:
-                logger.info(
-                    f"Multiclass classification: using SHAP values for class {class_of_interest}."
-                )
-                return shap_values[class_of_interest]
-            else:
-                raise ValueError(
-                    f"Invalid class_of_interest: {class_of_interest}. Must be between 0 and {num_classes - 1}."
-                )
-    else:
-        # For regression, return the SHAP values as they are
-        # TODO: shap_values is always an array of all predictions, how is the if above supposed to work?
-        # logger.info("Regression model detected: returning SHAP values as-is.")
-        return shap_values
+    return (
+        generate_shap_plot("mean", shap_values, shap_sample),
+        generate_shap_plot("summary", shap_values, shap_sample),
+    )

validmind/tests/model_validation/sklearn/SilhouettePlot.py

@@ -2,23 +2,17 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import matplotlib.pyplot as plt
 import numpy as np
 from sklearn.metrics import silhouette_samples, silhouette_score
 
-from validmind.vm_models import (
-    Figure,
-    Metric,
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-)
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class SilhouettePlot(Metric):
+@tags("sklearn", "model_performance")
+@tasks("clustering")
+def SilhouettePlot(model: VMModel, dataset: VMDataset):
     """
     Calculates and visualizes Silhouette Score, assessing the degree of data point suitability to its cluster in ML
     models.
@@ -65,93 +59,54 @@ class SilhouettePlot(Metric):
     assignment nuances, so potentially relevant details may be omitted.
     - Computationally expensive for large datasets, as it requires pairwise distance computations.
     """
-
-    name = "silhouette_plot"
-    required_inputs = ["model", "dataset"]
-    tasks = ["clustering"]
-    tags = [
-        "sklearn",
-        "model_performance",
-    ]
-
-    def run(self):
-        y_pred_train = self.inputs.dataset.y_pred(self.inputs.model)
-        # Calculate the silhouette score
-        silhouette_avg = silhouette_score(
-            self.inputs.dataset.x,
-            y_pred_train,
-            metric="euclidean",
-        )
-        num_clusters = len(np.unique(y_pred_train))
-        # Calculate silhouette coefficients for each data point
-        sample_silhouette_values = silhouette_samples(
-            self.inputs.dataset.x, y_pred_train
-        )
-        # Create a silhouette plot
-        fig, ax = plt.subplots()
-        y_lower = 10
-
-        for i in range(num_clusters):
-            # Aggregate the silhouette scores for samples belonging to cluster i
-            ith_cluster_silhouette_values = sample_silhouette_values[y_pred_train == i]
-            ith_cluster_silhouette_values.sort()
-
-            size_cluster_i = ith_cluster_silhouette_values.shape[0]
-            y_upper = y_lower + size_cluster_i
-            color = plt.cm.viridis(float(i) / num_clusters)
-            ax.fill_betweenx(
-                np.arange(y_lower, y_upper),
-                0,
-                ith_cluster_silhouette_values,
-                facecolor=color,
-                edgecolor=color,
-                alpha=0.7,
-            )
-
-            # Label the silhouette plots with their cluster numbers at the middle
-            ax.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
-            # Compute the new y_lower for the next plot
-            y_lower = y_upper + 10
-
-        ax.set_title("Silhouette Plot for Clusters")
-        ax.set_xlabel("Silhouette Coefficient Values")
-        ax.set_ylabel("Cluster Label")
-
-        # The vertical line represents the average silhouette score
-        ax.axvline(x=silhouette_avg, color="red", linestyle="--")
-
-        figures = [
-            Figure(
-                for_object=self,
-                key=self.key,
-                figure=fig,
-            )
-        ]
-        # Close the figure to prevent it from displaying
-        plt.close(fig)
-
-        return self.cache_results(
-            metric_value={
-                "silhouette_score": {
-                    "silhouette_score": silhouette_avg,
-                },
-            },
-            figures=figures,
+    y_pred = dataset.y_pred(model)
+
+    silhouette_avg = silhouette_score(
+        X=dataset.x,
+        labels=y_pred,
+        metric="euclidean",
+    )
+
+    # Calculate silhouette coefficients for each data point
+    sample_silhouette_values = silhouette_samples(dataset.x, y_pred)
+    # Create a silhouette plot
+    fig, ax = plt.subplots()
+
+    y_lower = 10
+    num_clusters = len(np.unique(y_pred))
+    for i in range(num_clusters):
+        # Aggregate the silhouette scores for samples belonging to cluster i
+        ith_cluster_silhouette_values = sample_silhouette_values[y_pred == i]
+        ith_cluster_silhouette_values.sort()
+
+        size_cluster_i = ith_cluster_silhouette_values.shape[0]
+        y_upper = y_lower + size_cluster_i
+        color = plt.cm.viridis(float(i) / num_clusters)
+        ax.fill_betweenx(
+            np.arange(y_lower, y_upper),
+            0,
+            ith_cluster_silhouette_values,
+            facecolor=color,
+            edgecolor=color,
+            alpha=0.7,
         )
 
-    def summary(self, metric_value):
-        """
-        Build one table for summarizing the Silhouette score results
-        """
-        silhouette_score = metric_value["silhouette_score"]
-
-        table = []
-        table.append(silhouette_score)
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=table,
-                    metadata=ResultTableMetadata(title="Silhouette Score"),
-                ),
-            ]
-        )
+        # Label the silhouette plots with their cluster numbers at the middle
+        ax.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
+        # Compute the new y_lower for the next plot
+        y_lower = y_upper + 10
+
+    ax.set_title("Silhouette Plot for Clusters")
+    ax.set_xlabel("Silhouette Coefficient Values")
+    ax.set_ylabel("Cluster Label")
+
+    # The vertical line represents the average silhouette score
+    ax.axvline(x=silhouette_avg, color="red", linestyle="--")
+
+    plt.close()
+
+    return [
+        {
+            "Silhouette Score": silhouette_avg,
+        },
+    ], fig