validmind 2.5.25__py3-none-any.whl → 2.6.8__py3-none-any.whl

validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py

@@ -2,20 +2,33 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
+from typing import Union
 
 import plotly.graph_objects as go
 from sklearn.inspection import permutation_importance
 
+from validmind import tags, tasks
 from validmind.errors import SkipTestError
 from validmind.logging import get_logger
-from validmind.vm_models import Figure, Metric
+from validmind.vm_models import VMDataset, VMModel
 
 logger = get_logger(__name__)
 
 
-@dataclass
-class PermutationFeatureImportance(Metric):
+@tags(
+    "sklearn",
+    "binary_classification",
+    "multiclass_classification",
+    "feature_importance",
+    "visualization",
+)
+@tasks("classification", "text_classification")
+def PermutationFeatureImportance(
+    model: VMModel,
+    dataset: VMDataset,
+    fontsize: Union[int, None] = None,
+    figure_height: Union[int, None] = None,
+):
     """
     Assesses the significance of each feature in a model by evaluating the impact on model performance when feature
     values are randomly rearranged.
@@ -55,78 +68,47 @@ class PermutationFeatureImportance(Metric):
     allocate importance to one and not the other.
     - Cannot interact with certain libraries like statsmodels, pytorch, catboost, etc., thus limiting its applicability.
     """
-
-    name = "pfi"
-    required_inputs = ["model", "dataset"]
-    default_params = {
-        "fontsize": None,
-        "figure_height": 1000,
-    }
-    tasks = ["classification", "text_classification"]
-    tags = [
-        "sklearn",
-        "binary_classification",
-        "multiclass_classification",
-        "feature_importance",
-        "visualization",
-    ]
-
-    def run(self):
-        x = self.inputs.dataset.x_df()
-        y = self.inputs.dataset.y_df()
-
-        if self.inputs.model.library in [
-            "statsmodels",
-            "pytorch",
-            "catboost",
-            "transformers",
-            "R",
-        ]:
-            raise SkipTestError(f"Skipping PFI for {self.inputs.model.library} models")
-
-        pfi_values = permutation_importance(
-            self.inputs.model.model,
-            x,
-            y,
-            random_state=0,
-            n_jobs=-2,
-        )
-
-        pfi = {}
-        for i, column in enumerate(x.columns):
-            pfi[column] = [pfi_values["importances_mean"][i]], [
-                pfi_values["importances_std"][i]
-            ]
-
-        sorted_idx = pfi_values.importances_mean.argsort()
-
-        fig = go.Figure()
-        fig.add_trace(
-            go.Bar(
-                y=x.columns[sorted_idx],
-                x=pfi_values.importances[sorted_idx].mean(axis=1).T,
-                orientation="h",
-            )
-        )
-        fig.update_layout(
-            title_text="Permutation Importances",
-            yaxis=dict(
-                tickmode="linear",  # set tick mode to linear
-                dtick=1,  # set interval between ticks
-                tickfont=dict(
-                    size=self.params["fontsize"]
-                ),  # set the tick label font size
-            ),
-            height=self.params["figure_height"],  # use figure_height parameter here
-        )
-
-        return self.cache_results(
-            metric_value=pfi,
-            figures=[
-                Figure(
-                    for_object=self,
-                    key=f"pfi_{self.inputs.dataset.input_id}_{self.inputs.model.input_id}",
-                    figure=fig,
-                ),
-            ],
+    if model.library in [
+        "statsmodels",
+        "pytorch",
+        "catboost",
+        "transformers",
+        "R",
+    ]:
+        raise SkipTestError(f"Skipping PFI for {model.library} models")
+
+    pfi_values = permutation_importance(
+        estimator=model.model,
+        X=dataset.x_df(),
+        y=dataset.y_df(),
+        random_state=0,
+        n_jobs=-2,
+    )
+
+    pfi = {}
+    for i, column in enumerate(dataset.feature_columns):
+        pfi[column] = [pfi_values["importances_mean"][i]], [
+            pfi_values["importances_std"][i]
+        ]
+
+    sorted_idx = pfi_values.importances_mean.argsort()
+
+    fig = go.Figure()
+    fig.add_trace(
+        go.Bar(
+            y=[dataset.feature_columns[i] for i in sorted_idx],
+            x=pfi_values.importances[sorted_idx].mean(axis=1).T,
+            orientation="h",
         )
+    )
+    fig.update_layout(
+        title_text="Permutation Importances",
+        yaxis=dict(
+            tickmode="linear",
+            dtick=1,
+            tickfont=dict(size=fontsize),
+        ),
+        height=figure_height,
+    )
+
+    return fig

validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py

@@ -2,26 +2,87 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
+from typing import List
 
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
 from validmind.logging import get_logger
-from validmind.vm_models import (
-    Figure,
-    Metric,
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-)
+from validmind.vm_models import VMDataset, VMModel
 
 logger = get_logger(__name__)
 
 
-@dataclass
-class PopulationStabilityIndex(Metric):
+def calculate_psi(score_initial, score_new, num_bins=10, mode="fixed"):
+    """
+    Taken from:
+    https://towardsdatascience.com/checking-model-stability-and-population-shift-with-psi-and-csi-6d12af008783
+    """
+    eps = 1e-4
+
+    # Sort the data
+    score_initial.sort()
+    score_new.sort()
+
+    # Prepare the bins
+    min_val = min(min(score_initial), min(score_new))
+    max_val = max(max(score_initial), max(score_new))
+    if mode == "fixed":
+        bins = [
+            min_val + (max_val - min_val) * (i) / num_bins for i in range(num_bins + 1)
+        ]
+    elif mode == "quantile":
+        bins = pd.qcut(score_initial, q=num_bins, retbins=True)[
+            1
+        ]  # Create the quantiles based on the initial population
+    else:
+        raise ValueError(
+            f"Mode '{mode}' not recognized. Allowed options are 'fixed' and 'quantile'"
+        )
+    bins[0] = min_val - eps  # Correct the lower boundary
+    bins[-1] = max_val + eps  # Correct the higher boundary
+
+    # Bucketize the initial population and count the sample inside each bucket
+    bins_initial = pd.cut(score_initial, bins=bins, labels=range(1, num_bins + 1))
+    df_initial = pd.DataFrame({"initial": score_initial, "bin": bins_initial})
+    grp_initial = df_initial.groupby("bin").count()
+    grp_initial["percent_initial"] = grp_initial["initial"] / sum(
+        grp_initial["initial"]
+    )
+
+    # Bucketize the new population and count the sample inside each bucket
+    bins_new = pd.cut(score_new, bins=bins, labels=range(1, num_bins + 1))
+    df_new = pd.DataFrame({"new": score_new, "bin": bins_new})
+    grp_new = df_new.groupby("bin").count()
+    grp_new["percent_new"] = grp_new["new"] / sum(grp_new["new"])
+
+    # Compare the bins to calculate PSI
+    psi_df = grp_initial.join(grp_new, on="bin", how="inner")
+
+    # Add a small value for when the percent is zero
+    psi_df["percent_initial"] = psi_df["percent_initial"].apply(
+        lambda x: eps if x == 0 else x
+    )
+    psi_df["percent_new"] = psi_df["percent_new"].apply(lambda x: eps if x == 0 else x)
+
+    # Calculate the psi
+    psi_df["psi"] = (psi_df["percent_initial"] - psi_df["percent_new"]) * np.log(
+        psi_df["percent_initial"] / psi_df["percent_new"]
+    )
+
+    return psi_df.to_dict(orient="records")
+
+
+@tags(
+    "sklearn", "binary_classification", "multiclass_classification", "model_performance"
+)
+@tasks("classification", "text_classification")
+def PopulationStabilityIndex(
+    datasets: List[VMDataset], model: VMModel, num_bins: int = 10, mode: str = "fixed"
+):
     """
     Assesses the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across
     different datasets.
@@ -72,150 +133,39 @@ class PopulationStabilityIndex(Metric):
     relationships between features and the target variable (concept drift), or both. However, distinguishing between
     these causes is non-trivial.
     """
-
-    name = "psi"
-    required_inputs = ["model", "datasets"]
-    tasks = ["classification", "text_classification"]
-    tags = [
-        "sklearn",
-        "binary_classification",
-        "multiclass_classification",
-        "model_performance",
-    ]
-    default_params = {
-        "num_bins": 10,
-        "mode": "fixed",
-    }
-
-    def summary(self, metric_value):
-        # Add a table with the PSI values for each feature
-        # The data looks like this: [{"initial": 2652, "percent_initial": 0.5525, "new": 830, "percent_new": 0.5188, "psi": 0.0021},...
-        psi_table = [
-            {
-                "Bin": (
-                    i if i < (len(metric_value) - 1) else "Total"
-                ),  # The last bin is the "Total" bin
-                "Count Initial": values["initial"],
-                "Percent Initial (%)": values["percent_initial"] * 100,
-                "Count New": values["new"],
-                "Percent New (%)": values["percent_new"] * 100,
-                "PSI": values["psi"],
-            }
-            for i, values in enumerate(metric_value)
-        ]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=psi_table,
-                    metadata=ResultTableMetadata(
-                        title="Population Stability Index for Training and Test Datasets"
-                    ),
-                ),
-            ]
-        )
-
-    def _get_psi(
-        self, score_initial, score_new, num_bins=10, mode="fixed", as_dict=False
-    ):
-        """
-        Taken from:
-        https://towardsdatascience.com/checking-model-stability-and-population-shift-with-psi-and-csi-6d12af008783
-        """
-        eps = 1e-4
-
-        # Sort the data
-        score_initial.sort()
-        score_new.sort()
-
-        # Prepare the bins
-        min_val = min(min(score_initial), min(score_new))
-        max_val = max(max(score_initial), max(score_new))
-        if mode == "fixed":
-            bins = [
-                min_val + (max_val - min_val) * (i) / num_bins
-                for i in range(num_bins + 1)
-            ]
-        elif mode == "quantile":
-            bins = pd.qcut(score_initial, q=num_bins, retbins=True)[
-                1
-            ]  # Create the quantiles based on the initial population
-        else:
-            raise ValueError(
-                f"Mode '{mode}' not recognized. Allowed options are 'fixed' and 'quantile'"
-            )
-        bins[0] = min_val - eps  # Correct the lower boundary
-        bins[-1] = max_val + eps  # Correct the higher boundary
-
-        # Bucketize the initial population and count the sample inside each bucket
-        bins_initial = pd.cut(score_initial, bins=bins, labels=range(1, num_bins + 1))
-        df_initial = pd.DataFrame({"initial": score_initial, "bin": bins_initial})
-        grp_initial = df_initial.groupby("bin").count()
-        grp_initial["percent_initial"] = grp_initial["initial"] / sum(
-            grp_initial["initial"]
-        )
-
-        # Bucketize the new population and count the sample inside each bucket
-        bins_new = pd.cut(score_new, bins=bins, labels=range(1, num_bins + 1))
-        df_new = pd.DataFrame({"new": score_new, "bin": bins_new})
-        grp_new = df_new.groupby("bin").count()
-        grp_new["percent_new"] = grp_new["new"] / sum(grp_new["new"])
-
-        # Compare the bins to calculate PSI
-        psi_df = grp_initial.join(grp_new, on="bin", how="inner")
-
-        # Add a small value for when the percent is zero
-        psi_df["percent_initial"] = psi_df["percent_initial"].apply(
-            lambda x: eps if x == 0 else x
-        )
-        psi_df["percent_new"] = psi_df["percent_new"].apply(
-            lambda x: eps if x == 0 else x
-        )
-
-        # Calculate the psi
-        psi_df["psi"] = (psi_df["percent_initial"] - psi_df["percent_new"]) * np.log(
-            psi_df["percent_initial"] / psi_df["percent_new"]
-        )
-
-        return psi_df.to_dict(orient="records")
-
-    def run(self):
-        if self.inputs.model.library in ["statsmodels", "pytorch", "catboost"]:
-            logger.info(f"Skiping PSI for {self.inputs.model.library} models")
-            return
-
-        num_bins = self.params["num_bins"]
-        mode = self.params["mode"]
-
-        psi_results = self._get_psi(
-            self.inputs.model.predict_proba(self.inputs.datasets[0].x).copy(),
-            self.inputs.model.predict_proba(self.inputs.datasets[1].x).copy(),
-            num_bins=num_bins,
-            mode=mode,
-        )
-
-        trace1 = go.Bar(
-            x=list(range(len(psi_results))),
-            y=[d["percent_initial"] for d in psi_results],
-            name="Initial",
-            marker=dict(color="#DE257E"),
-        )
-        trace2 = go.Bar(
-            x=list(range(len(psi_results))),
-            y=[d["percent_new"] for d in psi_results],
-            name="New",
-            marker=dict(color="#E8B1F8"),
-        )
-
-        trace3 = go.Scatter(
-            x=list(range(len(psi_results))),
-            y=[d["psi"] for d in psi_results],
-            name="PSI",
-            yaxis="y2",
-            line=dict(color="#257EDE"),
-        )
-
-        layout = go.Layout(
+    if model.library in ["statsmodels", "pytorch", "catboost"]:
+        raise SkipTestError(f"Skiping PSI for {model.library} models")
+
+    psi_results = calculate_psi(
+        datasets[0].y_prob(model).copy(),
+        datasets[1].y_prob(model).copy(),
+        num_bins=num_bins,
+        mode=mode,
+    )
+
+    fig = go.Figure(
+        data=[
+            go.Bar(
+                x=list(range(len(psi_results))),
+                y=[d["percent_initial"] for d in psi_results],
+                name="Initial",
+                marker=dict(color="#DE257E"),
+            ),
+            go.Bar(
+                x=list(range(len(psi_results))),
+                y=[d["percent_new"] for d in psi_results],
+                name="New",
+                marker=dict(color="#E8B1F8"),
+            ),
+            go.Scatter(
+                x=list(range(len(psi_results))),
+                y=[d["psi"] for d in psi_results],
+                name="PSI",
+                yaxis="y2",
+                line=dict(color="#257EDE"),
+            ),
+        ],
+        layout=go.Layout(
             title="Population Stability Index (PSI) Plot",
             xaxis=dict(title="Bin"),
             yaxis=dict(title="Population Ratio"),
@@ -229,23 +179,31 @@ class PopulationStabilityIndex(Metric):
                 ],  # Adjust as needed
             ),
             barmode="group",
-        )
-
-        fig = go.Figure(data=[trace1, trace2, trace3], layout=layout)
-        figure = Figure(
-            for_object=self,
-            key=self.key,
-            figure=fig,
-        )
-
-        # Calculate the sum of each numeric column
-        total_psi = {
-            key: sum(d.get(key, 0) for d in psi_results)
-            for key in psi_results[0].keys()
-            if isinstance(psi_results[0][key], (int, float))
-        }
+        ),
+    )
+
+    # sum up the PSI values to get the total values
+    total_psi = {
+        key: sum(d.get(key, 0) for d in psi_results)
+        for key in psi_results[0].keys()
+        if isinstance(psi_results[0][key], (int, float))
+    }
+    psi_results.append(total_psi)
 
-        # Add the total PSI dictionary to the list
-        psi_results.append(total_psi)
+    table_title = f"Population Stability Index for {datasets[0].input_id} and {datasets[1].input_id} Datasets"
 
-        return self.cache_results(metric_value=psi_results, figures=[figure])
+    return {
+        table_title: [
+            {
+                "Bin": (
+                    i if i < (len(psi_results) - 1) else "Total"
+                ),  # The last bin is the "Total" bin
+                "Count Initial": values["initial"],
+                "Percent Initial (%)": values["percent_initial"] * 100,
+                "Count New": values["new"],
+                "Percent New (%)": values["percent_new"] * 100,
+                "PSI": values["psi"],
+            }
+            for i, values in enumerate(psi_results)
+        ],
+    }, fig

validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py

@@ -2,19 +2,19 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
 import plotly.graph_objects as go
 from sklearn.metrics import precision_recall_curve
 
+from validmind import tags, tasks
 from validmind.errors import SkipTestError
 from validmind.models import FoundationModel
-from validmind.vm_models import Figure, Metric
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class PrecisionRecallCurve(Metric):
+@tags("sklearn", "binary_classification", "model_performance", "visualization")
+@tasks("classification", "text_classification")
+def PrecisionRecallCurve(model: VMModel, dataset: VMDataset):
     """
     Evaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve.
 
@@ -55,59 +55,30 @@ class PrecisionRecallCurve(Metric):
     - It may not fully represent the overall accuracy of the model if the cost of false positives and false negatives
     are extremely different, or if the dataset is heavily imbalanced.
     """
+    if isinstance(model, FoundationModel):
+        raise SkipTestError("Skipping PrecisionRecallCurve for Foundation models")
 
-    name = "pr_curve"
-    required_inputs = ["model", "dataset"]
-    tasks = ["classification", "text_classification"]
-    tags = [
-        "sklearn",
-        "binary_classification",
-        "multiclass_classification",
-        "model_performance",
-        "visualization",
-    ]
-
-    def run(self):
-        if isinstance(self.inputs.model, FoundationModel):
-            raise SkipTestError("Skipping PrecisionRecallCurve for Foundation models")
-
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_prob(self.inputs.model)
-
-        # PR curve is only supported for binary classification
-        if len(np.unique(y_true)) > 2:
-            raise SkipTestError(
-                "Precision Recall Curve is only supported for binary classification models"
-            )
+    y_true = dataset.y
+    if len(np.unique(y_true)) > 2:
+        raise SkipTestError(
+            "Precision Recall Curve is only supported for binary classification models"
+        )
 
-        precision, recall, pr_thresholds = precision_recall_curve(y_true, y_pred)
+    precision, recall, _ = precision_recall_curve(y_true, dataset.y_prob(model))
 
-        trace = go.Scatter(
-            x=recall,
-            y=precision,
-            mode="lines",
-            name="Precision-Recall Curve",
-            line=dict(color="#DE257E"),
-        )
-        layout = go.Layout(
+    return go.Figure(
+        data=[
+            go.Scatter(
+                x=recall,
+                y=precision,
+                mode="lines",
+                name="Precision-Recall Curve",
+                line=dict(color="#DE257E"),
+            )
+        ],
+        layout=go.Layout(
             title="Precision-Recall Curve",
             xaxis=dict(title="Recall"),
             yaxis=dict(title="Precision"),
-        )
-
-        fig = go.Figure(data=[trace], layout=layout)
-
-        return self.cache_results(
-            metric_value={
-                "precision": precision,
-                "recall": recall,
-                "thresholds": pr_thresholds,
-            },
-            figures=[
-                Figure(
-                    for_object=self,
-                    key="pr_curve",
-                    figure=fig,
-                )
-            ],
-        )
+        ),
+    )