validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/model_validation/sklearn/AdjustedRandIndex.py

@@ -2,15 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
+from sklearn.metrics import adjusted_rand_score
 
-from sklearn import metrics
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
-from .ClusterPerformance import ClusterPerformance
 
-
-@dataclass
-class AdjustedRandIndex(ClusterPerformance):
+@tags("sklearn", "model_performance", "clustering")
+@tasks("clustering")
+def AdjustedRandIndex(model: VMModel, dataset: VMDataset):
     """
     Measures the similarity between two data clusters using the Adjusted Rand Index (ARI) metric in clustering machine
     learning models.
@@ -49,14 +49,11 @@ class AdjustedRandIndex(ClusterPerformance):
     - It may be difficult to interpret the implications of an ARI score without context or a benchmark, as it is
     heavily dependent on the characteristics of the dataset used.
     """
-
-    name = "adjusted_rand_index"
-    required_inputs = ["model", "dataset"]
-    tasks = ["clustering"]
-    tags = [
-        "sklearn",
-        "model_performance",
+    return [
+        {
+            "Adjusted Rand Index": adjusted_rand_score(
+                labels_true=dataset.y,
+                labels_pred=dataset.y_pred(model),
+            )
+        }
     ]
-
-    def metric_info(self):
-        return {"Adjusted Rand Index": metrics.adjusted_rand_score}

validmind/tests/model_validation/sklearn/ClassifierPerformance.py

@@ -2,24 +2,25 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
 from sklearn.metrics import classification_report, roc_auc_score
 from sklearn.preprocessing import LabelBinarizer
 
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
 def multiclass_roc_auc_score(y_test, y_pred, average="macro"):
     lb = LabelBinarizer()
     lb.fit(y_test)
-
     return roc_auc_score(lb.transform(y_test), lb.transform(y_pred), average=average)
 
 
-@dataclass
-class ClassifierPerformance(Metric):
+@tags(
+    "sklearn", "binary_classification", "multiclass_classification", "model_performance"
+)
+@tasks("classification", "text_classification")
+def ClassifierPerformance(dataset: VMDataset, model: VMModel, average: str = "macro"):
     """
     Evaluates performance of binary or multiclass classification models using precision, recall, F1-Score, accuracy,
     and ROC AUC scores.
@@ -57,92 +58,53 @@ class ClassifierPerformance(Metric):
     - Specifically designed for classification models and not suitable for regression models.
     - May provide limited insights if the test dataset does not represent real-world scenarios adequately.
     """
-
-    name = "classifier_performance"
-    required_inputs = ["model", "dataset"]
-    tasks = ["classification", "text_classification"]
-    tags = [
-        "sklearn",
-        "binary_classification",
-        "multiclass_classification",
-        "model_performance",
+    y_pred = dataset.y_pred(model)
+    y_true = dataset.y
+
+    labels = np.unique(y_true)
+    labels = sorted(labels.tolist())
+
+    report = classification_report(
+        y_true=y_true,
+        y_pred=y_pred,
+        output_dict=True,
+        zero_division=0,
+    )
+
+    if len(labels) > 2:
+        y_true = y_true.astype(y_pred.dtype)
+        roc_auc = multiclass_roc_auc_score(y_true, y_pred, average=average)
+    else:
+        y_prob = dataset.y_prob(model)
+        y_true = y_true.astype(y_prob.dtype).flatten()
+        roc_auc = roc_auc_score(y_true, y_prob, average=average)
+
+    report["roc_auc"] = roc_auc
+
+    pr_f1_table = [
+        {
+            "Class": f"{class_name}",
+            "Precision": report[f"{class_name}"]["precision"],
+            "Recall": report[f"{class_name}"]["recall"],
+            "F1": report[f"{class_name}"]["f1-score"],
+        }
+        for class_name in labels
     ]
-    default_params = {"average": "macro"}
-
-    def summary(self, metric_value: dict):
-        """
-        When building a multi-class summary we need to calculate weighted average,
-        macro average and per class metrics.
-        """
-        classes = {str(i) for i in np.unique(self.inputs.dataset.y)}
-        pr_f1_table = [
-            {
-                "Class": class_name,
-                "Precision": metric_value[class_name]["precision"],
-                "Recall": metric_value[class_name]["recall"],
-                "F1": metric_value[class_name]["f1-score"],
-            }
-            for class_name in classes
-        ]
-        pr_f1_table.extend(
-            [
-                {
-                    "Class": "Weighted Average",
-                    "Precision": metric_value["weighted avg"]["precision"],
-                    "Recall": metric_value["weighted avg"]["recall"],
-                    "F1": metric_value["weighted avg"]["f1-score"],
-                },
-                {
-                    "Class": "Macro Average",
-                    "Precision": metric_value["macro avg"]["precision"],
-                    "Recall": metric_value["macro avg"]["recall"],
-                    "F1": metric_value["macro avg"]["f1-score"],
-                },
-            ]
-        )
 
-        acc_roc_auc_table = [
+    for avg in ["weighted avg", "macro avg"]:
+        pr_f1_table.append(
             {
-                "Metric": "Accuracy" if metric_name == "accuracy" else "ROC AUC",
-                "Value": metric_value[metric_name],
+                "Class": avg.replace("avg", "Average").title(),
+                "Precision": report[avg]["precision"],
+                "Recall": report[avg]["recall"],
+                "F1": report[avg]["f1-score"],
             }
-            for metric_name in ["accuracy", "roc_auc"]
-        ]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=pr_f1_table,
-                    metadata=ResultTableMetadata(title="Precision, Recall, and F1"),
-                ),
-                ResultTable(
-                    data=acc_roc_auc_table,
-                    metadata=ResultTableMetadata(title="Accuracy and ROC AUC"),
-                ),
-            ]
         )
 
-    def run(self):
-        report = classification_report(
-            self.inputs.dataset.y,
-            self.inputs.dataset.y_pred(self.inputs.model),
-            output_dict=True,
-            zero_division=0,
-        )
-
-        y_true = self.inputs.dataset.y
-
-        if len(np.unique(y_true)) > 2:
-            y_pred = self.inputs.dataset.y_pred(self.inputs.model)
-            y_true = y_true.astype(y_pred.dtype)
-            roc_auc = multiclass_roc_auc_score(
-                y_true, y_pred, average=self.params["average"]
-            )
-        else:
-            y_prob = self.inputs.dataset.y_prob(self.inputs.model)
-            y_true = y_true.astype(y_prob.dtype).flatten()
-            roc_auc = roc_auc_score(y_true, y_prob, average=self.params["average"])
-
-        report["roc_auc"] = roc_auc
-
-        return self.cache_results(report)
+    return {
+        "Precision, Recall, and F1": pr_f1_table,
+        "Accuracy and ROC AUC": [
+            {"Metric": m, "Value": report[k]}
+            for m, k in [("Accuracy", "accuracy"), ("ROC AUC", "roc_auc")]
+        ],
+    }

validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py

@@ -2,17 +2,17 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
-import pandas as pd
 from sklearn.metrics.pairwise import cosine_similarity
 
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class ClusterCosineSimilarity(Metric):
+@tags("sklearn", "model_performance", "clustering")
+@tasks("clustering")
+def ClusterCosineSimilarity(model: VMModel, dataset: VMDataset):
     """
     Measures the intra-cluster similarity of a clustering model using cosine similarity.
 
@@ -56,59 +56,29 @@ class ClusterCosineSimilarity(Metric):
     - Lastly, although rare, perfect perpendicular vectors (cosine similarity = 0) could be within the same cluster,
     which may give an inaccurate representation of a 'bad' cluster due to low cosine similarity score.
     """
-
-    name = "cluster_cosine_similarity"
-    required_inputs = ["model", "dataset"]
-    tasks = ["clustering"]
-    tags = [
-        "sklearn",
-        "model_performance",
-    ]
-
-    def run(self):
-        y_true_train = self.inputs.dataset.y
-        y_pred_train = self.inputs.dataset.y_pred(self.inputs.model)
-        y_true_train = y_true_train.astype(y_pred_train.dtype).flatten()
-        num_clusters = len(np.unique(y_pred_train))
-        # Calculate cosine similarity for each cluster
-        results = []
-        for cluster_id in range(num_clusters):
-            cluster_mask = y_pred_train == cluster_id
-            cluster_data = self.inputs.dataset.x[cluster_mask]
-            if cluster_data.size != 0:
-                # Compute the centroid of the cluster
-                cluster_centroid = np.mean(cluster_data, axis=0)
-                # Compute cosine similarities between the centroid and data points in the cluster
-                cosine_similarities = cosine_similarity(
-                    cluster_data, [cluster_centroid]
-                )
-                # Extract cosine similarity values for each data point in the cluster
-                cosine_similarities = cosine_similarities.flatten()
-                results.append(
-                    {
-                        "Cluster": cluster_id,
-                        "Mean Cosine Similarity": np.mean(cosine_similarities),
-                    }
-                )
-        return self.cache_results(
-            {
-                "cosine_similarity": pd.DataFrame(results).to_dict(orient="records"),
-            }
-        )
-
-    def summary(self, metric_value):
-        """
-        Build one table for summarizing the cluster cosine similarity results
-        """
-        summary_regression = metric_value["cosine_similarity"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=summary_regression,
-                    metadata=ResultTableMetadata(
-                        title="Cluster Cosine Similarity Results"
+    y_pred = dataset.y_pred(model)
+    num_clusters = len(np.unique(y_pred))
+
+    table = []
+
+    for cluster_idx in range(num_clusters):
+        cluster_data = dataset.x[y_pred == cluster_idx]
+
+        if cluster_data.size != 0:
+            cluster_centroid = np.mean(cluster_data, axis=0)
+            table.append(
+                {
+                    "Cluster": cluster_idx,
+                    "Mean Cosine Similarity": np.mean(
+                        cosine_similarity(
+                            X=cluster_data,
+                            Y=[cluster_centroid],
+                        ).flatten()
                     ),
-                ),
-            ]
-        )
+                }
+            )
+
+    if not table:
+        raise SkipTestError("No clusters found")
+
+    return table

validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py

@@ -2,17 +2,74 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
-from sklearn import metrics
-
-from validmind.vm_models import ResultSummary, ResultTable
-
-from .ClusterPerformance import ClusterPerformance
-
-
-@dataclass
-class ClusterPerformanceMetrics(ClusterPerformance):
+from sklearn.metrics import (
+    adjusted_mutual_info_score,
+    adjusted_rand_score,
+    completeness_score,
+    fowlkes_mallows_score,
+    homogeneity_score,
+    v_measure_score,
+)
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+HOMOGENEITY = """
+The homogeneity score is a clustering evaluation metric that quantifies the degree to which each cluster within a
+clustering solution contains only data points that belong to a single true class or category. It provides a score
+within the range of 0 to 1, where a higher homogeneity score indicates that the clusters are more pure and internally
+consistent with respect to the ground truth labels, meaning that the data points within each cluster are closely related
+in terms of their actual class membership.
+"""
+
+COMPLETENESS = """
+The completeness score is a clustering evaluation metric used to assess how well a clustering solution captures all data points
+that belong to a single true class or category. It quantifies the extent to which the data points of a given class are
+grouped into a single cluster. The completeness score ranges from 0 to 1, with a higher score indicating that the clustering
+solution effectively accounts for all data points within their actual class, emphasizing the comprehensiveness of the
+clustering results with respect to the ground truth labels.
+"""
+
+V_MEASURE = """
+The V-Measure score is a clustering evaluation metric that combines both homogeneity and completeness to provide a
+single measure of the overall quality of a clustering solution. It takes into account how well clusters are internally
+coherent (homogeneity) and how well they capture all data points from the true classes (completeness). The V-Measure
+score ranges from 0 to 1, where a higher score indicates a better clustering result. It balances the trade-off between
+cluster purity and the extent to which all data points from true classes are captured, offering a comprehensive evaluation
+of the clustering performance.
+"""
+ADJUSTED_RAND_INDEX = """
+The Adjusted Rand Index (ARI) is a clustering evaluation metric used to measure the
+similarity between the cluster assignments in a clustering solution and the true class labels. It calculates a
+score that ranges from -1 to 1, with a higher score indicating a better clustering result. A score of 1 signifies
+perfect agreement between the clustering and the ground truth, while a score near 0 implies that the clustering
+is random with respect to the true labels, and negative values indicate disagreement. ARI accounts for chance
+clustering, making it a robust measure for assessing the quality of clustering solutions by considering both the
+extent of agreement and potential randomness in the assignments.
+"""
+
+ADJUSTED_MUTUAL_INFORMATION = """
+The Adjusted Mutual Information (AMI) is a clustering evaluation metric used to quantify the degree of
+agreement between a clustering solution and the true class labels. It provides a score that ranges from 0 to 1,
+with a higher score indicating a better clustering result. A score of 1 signifies perfect agreement,
+while a score of 0 suggests that the clustering is random with respect to the true labels. AMI takes into account the
+potential randomness in the assignments and adjusts for chance, making it a robust measure that considers both the
+extent of agreement and the potential for random clustering.
+"""
+
+FOULKES_MALLOWS_SCORE = """
+The Fowlkes-Mallows score is a clustering evaluation metric used to assess the quality of
+a clustering solution by measuring the geometric mean of two fundamental clustering metrics: precision and recall. It
+provides a score that ranges from 0 to 1, where a higher score indicates a better clustering result. A score of 1 signifies
+perfect agreement with the true class labels, while lower scores suggest less precise and recall clustering performance.
+The Fowlkes-Mallows score offers a balanced evaluation of clustering quality by considering both the ability to correctly
+identify members of the same class (precision) and the ability to capture all members of the same class (recall).
+"""
+
+
+@tags("sklearn", "model_performance", "clustering")
+@tasks("clustering")
+def ClusterPerformanceMetrics(model: VMModel, dataset: VMDataset):
     """
     Evaluates the performance of clustering machine learning models using multiple established metrics.
 
@@ -58,75 +115,53 @@ class ClusterPerformanceMetrics(ClusterPerformance):
     - Does not consider aspects like computational efficiency of the model or its capability to handle high dimensional
     data.
     """
-
-    name = "homogeneity_score"
-    required_inputs = ["model", "dataset"]
-    tasks = ["clustering"]
-    tags = ["sklearn", "model_performance"]
-    default_metrics = {
-        "Homogeneity Score": metrics.homogeneity_score,
-        "Completeness Score": metrics.completeness_score,
-        "V Measure": metrics.v_measure_score,
-        "Adjusted Rand Index": metrics.adjusted_rand_score,
-        "Adjusted Mutual Information": metrics.adjusted_mutual_info_score,
-        "Fowlkes-Mallows score": metrics.fowlkes_mallows_score,
-    }
-    default_metrics_desc = {
-        "Homogeneity Score": """The homogeneity score is a clustering evaluation metric that quantifies
-        the degree to which each cluster within a clustering solution contains only data points that belong
-        to a single true class or category. It provides a score within the range of 0 to 1, where a higher
-        homogeneity score indicates that the clusters are more pure and internally consistent with respect
-        to the ground truth labels, meaning that the data points within each cluster are closely related in
-        terms of their actual class membership.
-        """,
-        "Completeness Score": """The completeness score is a clustering evaluation metric used to assess how
-        well a clustering solution captures all data points that belong to a single true class or category.
-        It quantifies the extent to which the data points of a given class are grouped into a single cluster.
-        The completeness score ranges from 0 to 1, with a higher score indicating that the clustering solution
-        effectively accounts for all data points within their actual class, emphasizing the comprehensiveness of
-        the clustering results with respect to the ground truth labels.""",
-        "V Measure": """The V-Measure score is a clustering evaluation metric that combines both homogeneity and
-        completeness to provide a single measure of the overall quality of a clustering solution. It takes into
-        account how well clusters are internally coherent (homogeneity) and how well they capture all data points
-        from the true classes (completeness). The V-Measure score ranges from 0 to 1, where a higher score indicates
-        a better clustering result. It balances the trade-off between cluster purity and the extent to which all data
-        points from true classes are captured, offering a comprehensive evaluation of the clustering performance.""",
-        "Adjusted Rand Index": """The Adjusted Rand Index (ARI) is a clustering evaluation metric used to measure the
-        similarity between the cluster assignments in a clustering solution and the true class labels. It calculates a
-        score that ranges from -1 to 1, with a higher score indicating a better clustering result. A score of 1 signifies
-        perfect agreement between the clustering and the ground truth, while a score near 0 implies that the clustering
-        is random with respect to the true labels, and negative values indicate disagreement. ARI accounts for chance
-        clustering, making it a robust measure for assessing the quality of clustering solutions by considering both the
-        extent of agreement and potential randomness in the assignments.""",
-        "Adjusted Mutual Information": """The Adjusted Mutual Information (AMI) is a clustering evaluation metric used to
-        quantify the degree of agreement between a clustering solution and the true class labels. It provides a score that
-        ranges from 0 to 1, with a higher score indicating a better clustering result. A score of 1 signifies perfect agreement,
-        while a score of 0 suggests that the clustering is random with respect to the true labels. AMI takes into account the
-        potential randomness in the assignments and adjusts for chance, making it a robust measure that considers both the
-        extent of agreement and the potential for random clustering.""",
-        "Fowlkes-Mallows score": """The Fowlkes-Mallows score is a clustering evaluation metric used to assess the quality of
-        a clustering solution by measuring the geometric mean of two fundamental clustering metrics: precision and recall. It
-        provides a score that ranges from 0 to 1, where a higher score indicates a better clustering result. A score of 1 signifies
-        perfect agreement with the true class labels, while lower scores suggest less precise and recall clustering performance.
-        The Fowlkes-Mallows score offers a balanced evaluation of clustering quality by considering both the ability to correctly
-        identify members of the same class (precision) and the ability to capture all members of the same class (recall).""",
-    }
-
-    def summary(self, raw_results):
-        """
-        Returns a summarized representation of the dataset split information
-        """
-        table_records = []
-        for result in raw_results:
-            for key, _ in result.items():
-                table_records.append(
-                    {
-                        "Description": self.default_metrics_desc[key],
-                        key: result[key],
-                    }
-                )
-
-        return ResultSummary(results=[ResultTable(data=table_records)])
-
-    def metric_info(self):
-        return self.default_metrics
+    return [
+        {
+            "Metric": "Homogeneity Score",
+            "Description": HOMOGENEITY,
+            "Value": homogeneity_score(
+                labels_true=dataset.y,
+                labels_pred=dataset.y_pred(model),
+            ),
+        },
+        {
+            "Metric": "Completeness Score",
+            "Description": COMPLETENESS,
+            "Value": completeness_score(
+                labels_true=dataset.y,
+                labels_pred=dataset.y_pred(model),
+            ),
+        },
+        {
+            "Metric": "V Measure",
+            "Description": V_MEASURE,
+            "Value": v_measure_score(
+                labels_true=dataset.y,
+                labels_pred=dataset.y_pred(model),
+            ),
+        },
+        {
+            "Metric": "Adjusted Rand Index",
+            "Description": ADJUSTED_RAND_INDEX,
+            "Value": adjusted_rand_score(
+                labels_true=dataset.y,
+                labels_pred=dataset.y_pred(model),
+            ),
+        },
+        {
+            "Metric": "Adjusted Mutual Information",
+            "Description": ADJUSTED_MUTUAL_INFORMATION,
+            "Value": adjusted_mutual_info_score(
+                labels_true=dataset.y,
+                labels_pred=dataset.y_pred(model),
+            ),
+        },
+        {
+            "Metric": "Fowlkes-Mallows score",
+            "Description": FOULKES_MALLOWS_SCORE,
+            "Value": fowlkes_mallows_score(
+                labels_true=dataset.y,
+                labels_pred=dataset.y_pred(model),
+            ),
+        },
+    ]

validmind/tests/model_validation/sklearn/CompletenessScore.py

@@ -2,15 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
+from sklearn.metrics import completeness_score
 
-from sklearn import metrics
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
-from .ClusterPerformance import ClusterPerformance
 
-
-@dataclass
-class CompletenessScore(ClusterPerformance):
+@tags("sklearn", "model_performance", "clustering")
+@tasks("clustering")
+def CompletenessScore(model: VMModel, dataset: VMDataset):
     """
     Evaluates a clustering model's capacity to categorize instances from a single class into the same cluster.
 
@@ -47,14 +47,11 @@ class CompletenessScore(ClusterPerformance):
     - The Completeness Score only applies to clustering models; it cannot be used for other types of machine learning
     models.
     """
-
-    name = "homogeneity_score"
-    required_inputs = ["model", "dataset"]
-    tasks = ["clustering"]
-    tags = [
-        "sklearn",
-        "model_performance",
+    return [
+        {
+            "Completeness Score": completeness_score(
+                labels_true=dataset.y,
+                labels_pred=dataset.y_pred(model),
+            )
+        }
     ]
-
-    def metric_info(self):
-        return {"Completeness Score": metrics.completeness_score}