validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/model_validation/sklearn/ConfusionMatrix.py

@@ -2,17 +2,24 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
 
 import numpy as np
 import plotly.figure_factory as ff
-from sklearn import metrics
+from sklearn.metrics import confusion_matrix
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class ConfusionMatrix(Metric):
+@tags(
+    "sklearn",
+    "binary_classification",
+    "multiclass_classification",
+    "model_performance",
+    "visualization",
+)
+@tasks("classification", "text_classification")
+def ConfusionMatrix(dataset: VMDataset, model: VMModel):
     """
     Evaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix
     heatmap.
@@ -59,95 +66,56 @@ class ConfusionMatrix(Metric):
     - Risks of misinterpretation exist because the matrix doesn't directly provide precision, recall, or F1-score data.
     These metrics have to be computed separately.
     """
+    y_pred = dataset.y_pred(model)
+    y_true = dataset.y.astype(y_pred.dtype)
 
-    name = "confusion_matrix"
-    required_inputs = ["model", "dataset"]
-    tasks = ["classification", "text_classification"]
-    tags = [
-        "sklearn",
-        "binary_classification",
-        "multiclass_classification",
-        "model_performance",
-        "visualization",
-    ]
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        labels = np.unique(y_true)
-        labels.sort()
-        labels = np.array(labels).T.tolist()
-
-        y_pred = self.inputs.dataset.y_pred(self.inputs.model)
-        y_true = y_true.astype(y_pred.dtype)
-
-        cm = metrics.confusion_matrix(y_true, y_pred, labels=labels)
-
-        text = None
-        if len(labels) == 2:
-            tn, fp, fn, tp = cm.ravel()
-
-            # Custom text to display on the heatmap cells
-            text = [
-                [
-                    f"<b>True Negatives (TN)</b><br />{tn}",
-                    f"<b>False Positives (FP)</b><br />{fp}",
-                ],
-                [
-                    f"<b>False Negatives (FN)</b><br />{fn}",
-                    f"<b>True Positives (TP)</b><br />{tp}",
-                ],
-            ]
-
-        fig = ff.create_annotated_heatmap(
-            z=cm,
-            colorscale="Blues",
-            x=labels,
-            y=labels,
-            annotation_text=text,
-        )
-
-        fig["data"][0][
-            "hovertemplate"
-        ] = "True Label:%{y}<br>Predicted Label:%{x}<br>Count:%{z}<extra></extra>"
-
-        fig.update_layout(
-            xaxis=dict(title="Predicted label"),
-            yaxis=dict(title="True label"),
-            autosize=False,
-            width=600,
-            height=600,
-        )
-
-        # Add an annotation at the bottom of the heatmap
-        fig.add_annotation(
-            x=0.5,
-            y=-0.1,
-            xref="paper",
-            yref="paper",
-            text=f"Confusion Matrix for {self.inputs.model.input_id} on {self.inputs.dataset.input_id}",
-            showarrow=False,
-            font=dict(size=14),
-        )
-
-        return self.cache_results(
-            metric_value={
-                "confusion_matrix": cm,
-            },
-            figures=[
-                Figure(
-                    for_object=self,
-                    key="confusion_matrix",
-                    figure=fig,
-                )
-            ],
-        )
-
-    def test(self):
-        """Unit Test for Confusion Matrix Metric"""
-        assert self.result is not None
+    labels = np.unique(y_true)
+    labels = sorted(labels.tolist())
 
-        assert self.result.metric is not None
-        assert isinstance(self.result.metric.value, dict)
-        assert "confusion_matrix" in self.result.metric.value
+    cm = confusion_matrix(y_true, y_pred, labels=labels)
 
-        assert len(self.result.figures) == 1
+    text = None
+    if len(labels) == 2:
+        tn, fp, fn, tp = cm.ravel()
+        text = [
+            [
+                f"<b>True Negatives (TN)</b><br />{tn}",
+                f"<b>False Positives (FP)</b><br />{fp}",
+            ],
+            [
+                f"<b>False Negatives (FN)</b><br />{fn}",
+                f"<b>True Positives (TP)</b><br />{tp}",
+            ],
+        ]
+
+    fig = ff.create_annotated_heatmap(
+        z=cm,
+        colorscale="Blues",
+        x=labels,
+        y=labels,
+        annotation_text=text,
+    )
+
+    fig["data"][0][
+        "hovertemplate"
+    ] = "True Label:%{y}<br>Predicted Label:%{x}<br>Count:%{z}<extra></extra>"
+
+    fig.update_layout(
+        xaxis=dict(title="Predicted label"),
+        yaxis=dict(title="True label"),
+        autosize=False,
+        width=600,
+        height=600,
+    )
+
+    fig.add_annotation(
+        x=0.5,
+        y=-0.1,
+        xref="paper",
+        yref="paper",
+        text=f"Confusion Matrix for {model.input_id} on {dataset.input_id}",
+        showarrow=False,
+        font=dict(size=14),
+    )
+
+    return fig

validmind/tests/model_validation/sklearn/FeatureImportance.py

@@ -6,11 +6,12 @@ import pandas as pd
 from sklearn.inspection import permutation_importance
 
 from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("model_explainability", "sklearn")
 @tasks("regression", "time_series_forecasting")
-def FeatureImportance(dataset, model, num_features=3):
+def FeatureImportance(dataset: VMDataset, model: VMModel, num_features: int = 3):
     """
     Compute feature importance scores for a given model and generate a summary table
     with the top important features.
@@ -53,20 +54,18 @@ def FeatureImportance(dataset, model, num_features=3):
     """
     results_list = []
 
-    x = dataset.x_df()
-    y = dataset.y_df()
-
     pfi_values = permutation_importance(
-        model.model,
-        x,
-        y,
+        estimator=model.model,
+        X=dataset.x_df(),
+        y=dataset.y_df(),
         random_state=0,
         n_jobs=-2,
     )
 
     # Create a dictionary to store PFI scores
     pfi = {
-        column: pfi_values["importances_mean"][i] for i, column in enumerate(x.columns)
+        column: pfi_values["importances_mean"][i]
+        for i, column in enumerate(dataset.feature_columns)
     }
 
     # Sort features by their importance

validmind/tests/model_validation/sklearn/FowlkesMallowsScore.py

@@ -2,15 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 from sklearn import metrics
 
-from .ClusterPerformance import ClusterPerformance
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class FowlkesMallowsScore(ClusterPerformance):
+@tags("sklearn", "model_performance")
+@tasks("clustering")
+def FowlkesMallowsScore(dataset: VMDataset, model: VMModel):
     """
     Evaluates the similarity between predicted and actual cluster assignments in a model using the Fowlkes-Mallows
     score.
@@ -52,14 +52,11 @@ class FowlkesMallowsScore(ClusterPerformance):
     - It does not handle mismatching numbers of clusters between the true and predicted labels. As such, it may return
     misleading results if the predicted labels suggest a different number of clusters than what is in the true labels.
     """
-
-    name = "fowlkes_mallows_score"
-    required_inputs = ["model", "dataset"]
-    tasks = ["clustering"]
-    tags = [
-        "sklearn",
-        "model_performance",
+    return [
+        {
+            "Fowlkes-Mallows score": metrics.fowlkes_mallows_score(
+                labels_true=dataset.y,
+                labels_pred=dataset.y_pred(model),
+            )
+        }
     ]
-
-    def metric_info(self):
-        return {"Fowlkes-Mallows score": metrics.fowlkes_mallows_score}

validmind/tests/model_validation/sklearn/HomogeneityScore.py

@@ -2,15 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 from sklearn import metrics
 
-from .ClusterPerformance import ClusterPerformance
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class HomogeneityScore(ClusterPerformance):
+@tags("sklearn", "model_performance")
+@tasks("clustering")
+def HomogeneityScore(dataset: VMDataset, model: VMModel):
     """
     Assesses clustering homogeneity by comparing true and predicted labels, scoring from 0 (heterogeneous) to 1
     (homogeneous).
@@ -50,14 +50,11 @@ class HomogeneityScore(ClusterPerformance):
     - The score does not address the actual number of clusters formed, or the evenness of cluster sizes. It only checks
     the homogeneity within the given clusters created by the model.
     """
-
-    name = "homogeneity_score"
-    required_inputs = ["model", "dataset"]
-    tasks = ["clustering"]
-    tags = [
-        "sklearn",
-        "model_performance",
+    return [
+        {
+            "Homogeneity Score": metrics.homogeneity_score(
+                labels_true=dataset.y,
+                labels_pred=dataset.y_pred(model),
+            )
+        }
     ]
-
-    def metric_info(self):
-        return {"Homogeneity Score": metrics.homogeneity_score}

validmind/tests/model_validation/sklearn/HyperParametersTuning.py

@@ -2,17 +2,23 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
+from typing import Union
 
-import pandas as pd
 from sklearn.model_selection import GridSearchCV
 
+from validmind import tags, tasks
 from validmind.errors import SkipTestError
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class HyperParametersTuning(Metric):
+@tags("sklearn", "model_performance")
+@tasks("classification", "clustering")
+def HyperParametersTuning(
+    model: VMModel,
+    dataset: VMDataset,
+    param_grid: Union[dict, None] = None,
+    scoring: Union[str, None] = None,
+):
     """
     Exerts exhaustive grid search to identify optimal hyperparameters for the model, improving performance.
 
@@ -54,51 +60,15 @@ class HyperParametersTuning(Metric):
     - There's a potential risk of overfitting the model if the training set is not representative of the data that the
     model will be applied to.
     """
-
-    name = "hyper_parameters_tuning"
-    required_inputs = ["model", "dataset"]
-    tasks = ["classification", "clustering"]
-    tags = ["sklearn", "model_performance"]
-    default_params = {"param_grid": None, "scoring": None}
-
-    def run(self):
-        param_grid = self.params["param_grid"]
-        if param_grid is None:
-            raise SkipTestError(
-                "param_grid in dictonary format must be provided to run this test"
-            )
-
-        model = self.inputs.model.model
-        estimators = GridSearchCV(
-            model, param_grid=param_grid, scoring=self.params["scoring"]
-        )
-        estimators.fit(self.inputs.dataset.x, self.inputs.dataset.y)
-
-        results = [
-            {
-                "Best Model": f"{estimators.best_estimator_}",
-                "Best Parameters": estimators.best_params_,
-            }
-        ]
-        return self.cache_results(
-            {
-                "parameters_tuning": pd.DataFrame(results).to_dict(orient="records"),
-            }
-        )
-
-    def summary(self, metric_value):
-        """
-        Build one table for summarizing the hyper parameters tunning
-        """
-        summary_regression = metric_value["parameters_tuning"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=summary_regression,
-                    metadata=ResultTableMetadata(
-                        title="Hyper Parameters Tuning Results"
-                    ),
-                ),
-            ]
-        )
+    if not param_grid:
+        raise SkipTestError("'param_grid' dictionary must be provided to run this test")
+
+    estimators = GridSearchCV(model.model, param_grid=param_grid, scoring=scoring)
+    estimators.fit(dataset.x, dataset.y)
+
+    return [
+        {
+            "Best Model": estimators.best_estimator_,
+            "Best Parameters": estimators.best_params_,
+        }
+    ]

validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py

@@ -2,20 +2,25 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
+from typing import List, Union
 
 import numpy as np
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 from scipy.spatial.distance import cdist
+from sklearn import clone
 from sklearn.metrics import silhouette_score
 
+from validmind import tags, tasks
 from validmind.errors import SkipTestError
-from validmind.vm_models import Figure, Metric
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class KMeansClustersOptimization(Metric):
+@tags("sklearn", "model_performance", "kmeans")
+@tasks("clustering")
+def KMeansClustersOptimization(
+    model: VMModel, dataset: VMDataset, n_clusters: Union[List[int], None] = None
+):
     """
     Optimizes the number of clusters in K-means models using Elbow and Silhouette methods.
 
@@ -62,80 +67,61 @@ class KMeansClustersOptimization(Metric):
     - Assumes spherical clusters (due to using the Euclidean distance in the Elbow method), which might not align with
     the actual structure of the data.
     """
-
-    name = "clusters_optimize_elbow_method"
-    required_inputs = ["model", "dataset"]
-    tasks = ["clustering"]
-    tags = ["sklearn", "model_performance", "kmeans"]
-
-    default_params = {"n_clusters": None}
-
-    def run(self):
-        n_clusters = self.params["n_clusters"]
-        if n_clusters is None:
-            raise SkipTestError("n_clusters parameter must be provide in list format")
-        model = self.inputs.model.model
-
-        distortions = {}
-        silhouette_avg = {}
-
-        for k in n_clusters:
-            # Building and fitting the model
-            kmeanModel = model.set_params(n_clusters=k)
-            kmeanModel = kmeanModel.fit(self.inputs.dataset.x)
-            # Calculate silhouette coefficients for each data point
-            silhouette_avg[k] = silhouette_score(
-                self.inputs.dataset.x,
-                kmeanModel.predict(self.inputs.dataset.x),
-            )
-
-            distortions[k] = (
-                sum(
-                    np.min(
-                        cdist(
-                            self.inputs.dataset.x,
-                            kmeanModel.cluster_centers_,
-                            "euclidean",
-                        ),
-                        axis=1,
-                    )
-                )
-                / self.inputs.dataset.x.shape[0]
-            )
-        fig = make_subplots(
-            rows=1,
-            cols=2,
-            subplot_titles=(
-                "The Silhouette value of each cluster",
-                "The Elbow Method using Distortion",
-            ),
+    if not n_clusters:
+        raise SkipTestError(
+            "Cluster range must be provided via the 'n_clusters' parameter"
         )
 
-        fig.add_trace(
-            go.Scatter(x=list(silhouette_avg.keys()), y=list(silhouette_avg.values())),
-            row=1,
-            col=1,
-        )
-        fig.update_xaxes(title_text="Number of clusters", row=1, col=1)
-        fig.update_yaxes(title_text="Avg Silhouette Score", row=1, col=1)
+    distortions = {}
+    silhouette_avg = {}
 
-        fig.add_trace(
-            go.Scatter(x=list(distortions.keys()), y=list(distortions.values())),
-            row=1,
-            col=2,
-        )
-        # Update xaxis properties
-        fig.update_xaxes(title_text="Number of clusters", showgrid=False, row=1, col=2)
-        fig.update_yaxes(title_text="Distortion", showgrid=False, row=1, col=2)
+    for k in n_clusters:
+        kmeanModel = clone(model.model).set_params(n_clusters=k).fit(dataset.x)
 
-        fig.update_layout(showlegend=False)
+        silhouette_avg[k] = silhouette_score(
+            dataset.x,
+            kmeanModel.predict(dataset.x),
+        )
 
-        figures = [
-            Figure(
-                for_object=self,
-                key=self.key,
-                figure=fig,
+        distortions[k] = (
+            sum(
+                np.min(
+                    cdist(
+                        dataset.x,
+                        kmeanModel.cluster_centers_,
+                        "euclidean",
+                    ),
+                    axis=1,
+                )
             )
-        ]
+            / dataset.x.shape[0]
+        )
 
-        return self.cache_results(figures=figures)
+    fig = make_subplots(
+        rows=1,
+        cols=2,
+        subplot_titles=(
+            "The Silhouette value of each cluster",
+            "The Elbow Method using Distortion",
+        ),
+    )
+
+    fig.add_trace(
+        go.Scatter(x=list(silhouette_avg.keys()), y=list(silhouette_avg.values())),
+        row=1,
+        col=1,
+    )
+    fig.update_xaxes(title_text="Number of clusters", row=1, col=1)
+    fig.update_yaxes(title_text="Avg Silhouette Score", row=1, col=1)
+
+    fig.add_trace(
+        go.Scatter(x=list(distortions.keys()), y=list(distortions.values())),
+        row=1,
+        col=2,
+    )
+    fig.update_xaxes(title_text="Number of clusters", showgrid=False, row=1, col=2)
+    fig.update_yaxes(title_text="Distortion", showgrid=False, row=1, col=2)
+
+    fig.update_layout(showlegend=False)
+
+    return fig

validmind/tests/model_validation/sklearn/MinimumAccuracy.py

@@ -1,24 +1,17 @@
 # Copyright © 2023-2024 ValidMind Inc. All rights reserved.
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+from sklearn.metrics import accuracy_score
 
-from dataclasses import dataclass
-from typing import List
+from validmind.tests import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
-import pandas as pd
-from sklearn import metrics
 
-from validmind.vm_models import (
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-    ThresholdTest,
-    ThresholdTestResult,
+@tags(
+    "sklearn", "binary_classification", "multiclass_classification", "model_performance"
 )
-
-
-@dataclass
-class MinimumAccuracy(ThresholdTest):
+@tasks("classification", "text_classification")
+def MinimumAccuracy(dataset: VMDataset, model: VMModel, min_threshold: float = 0.7):
     """
     Checks if the model's prediction accuracy meets or surpasses a specified threshold.
 
@@ -55,73 +48,12 @@ class MinimumAccuracy(ThresholdTest):
     - Inability to measure the model's precision, recall, or capacity to manage false positives or false negatives.
     - Focused on overall correctness and may not be sufficient for all types of model analytics.
     """
-
-    name = "accuracy_score"
-    required_inputs = ["model", "dataset"]
-    default_params = {"min_threshold": 0.7}
-    tasks = ["classification", "text_classification"]
-    tags = [
-        "sklearn",
-        "binary_classification",
-        "multiclass_classification",
-        "model_performance",
-    ]
-
-    def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        """
-        The accuracy score test returns results like these:
-        [{"values": {"score": 0.734375, "threshold": 0.7}, "passed": true}]
-        """
-        result = results[0]
-        results_table = [
-            {
-                "Score": result.values["score"],
-                "Threshold": result.values["threshold"],
-                "Pass/Fail": "Pass" if result.passed else "Fail",
-            }
-        ]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=pd.DataFrame(results_table),
-                    metadata=ResultTableMetadata(
-                        title="Minimum Accuracy Test on Test Data"
-                    ),
-                )
-            ]
-        )
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        class_pred = self.inputs.dataset.y_pred(self.inputs.model)
-        y_true = y_true.astype(class_pred.dtype)
-
-        accuracy_score = metrics.accuracy_score(y_true, class_pred)
-
-        passed = accuracy_score > self.params["min_threshold"]
-        results = [
-            ThresholdTestResult(
-                passed=passed,
-                values={
-                    "score": accuracy_score,
-                    "threshold": self.params["min_threshold"],
-                },
-            )
-        ]
-
-        return self.cache_results(results, passed=all([r.passed for r in results]))
-
-    def test(self):
-        # Test that there is a result and it's not None
-        assert self.result is not None
-        # Test that results are contained in a list
-        assert isinstance(self.result.test_results.results, list)
-        # Verify that there is exactly one result
-        assert len(self.result.test_results.results) == 1
-        # Extract the single result for clarity
-        test_result = self.result.test_results.results[0]
-        # Check the 'passed' condition logic against the test outcome
-        assert test_result.passed == (
-            test_result.values["score"] >= test_result.values["threshold"]
-        )
+    accuracy = accuracy_score(dataset.y, dataset.y_pred(model))
+
+    return [
+        {
+            "Score": accuracy,
+            "Threshold": min_threshold,
+            "Pass/Fail": "Pass" if accuracy > min_threshold else "Fail",
+        }
+    ], accuracy > min_threshold