validmind 2.5.25__py3-none-any.whl → 2.6.8__py3-none-any.whl

validmind/tests/model_validation/sklearn/TrainingTestDegradation.py

@@ -2,33 +2,26 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-from functools import partial
 from typing import List
 
-import pandas as pd
 from numpy import unique
-from sklearn import metrics, preprocessing
-
-from validmind.vm_models import (
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-    ThresholdTest,
-    ThresholdTestResult,
-)
-
+from sklearn.metrics import classification_report
 
-def multiclass_roc_auc_score(y_test, y_pred, average="macro"):
-    lb = preprocessing.LabelBinarizer()
-    lb.fit(y_test)
-    y_test = lb.transform(y_test)
-    y_pred = lb.transform(y_pred)
-    return metrics.roc_auc_score(y_test, y_pred, average=average)
+from validmind.tests import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class TrainingTestDegradation(ThresholdTest):
+@tags(
+    "sklearn",
+    "binary_classification",
+    "multiclass_classification",
+    "model_performance",
+    "visualization",
+)
+@tasks("classification", "text_classification")
+def TrainingTestDegradation(
+    datasets: List[VMDataset], model: VMModel, max_threshold: float = 0.10
+):
     """
     Tests if model performance degradation between training and test datasets exceeds a predefined threshold.
 
@@ -70,106 +63,39 @@ class TrainingTestDegradation(ThresholdTest):
     not always be available.
     - The test is currently only designed for classification tasks.
     """
-
-    name = "training_test_degradation"
-    required_inputs = ["model", "datasets"]
-
-    default_params = {
-        "metrics": ["accuracy", "precision", "recall", "f1"],
-        "max_threshold": 0.10,  # Maximum 10% degradation
-    }
-
-    tasks = ["classification", "text_classification"]
-    tags = [
-        "sklearn",
-        "binary_classification",
-        "multiclass_classification",
-        "model_performance",
-        "visualization",
-    ]
-
-    default_metrics = {
-        "accuracy": metrics.accuracy_score,
-        "precision": partial(metrics.precision_score, zero_division=0, average="micro"),
-        "recall": partial(metrics.recall_score, zero_division=0, average="micro"),
-        "f1": partial(metrics.f1_score, zero_division=0, average="micro"),
-    }
-
-    def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        """
-        The training test degradation test returns results like these:
-        [{"values":
-            {"test_score": 0.7225, "train_score": 0.7316666666666667, "degradation": 0.012528473804100214}, "test_name": "accuracy", "passed": true}, ...]
-        """
-        results_table = [
-            {
-                "Class": result.values["class"],
-                "Metric": result.test_name.title(),
-                "Train Score": result.values["train_score"],
-                "Test Score": result.values["test_score"],
-                "Degradation (%)": result.values["degradation"] * 100,
-                "Pass/Fail": "Pass" if result.passed else "Fail",
-            }
-            for result in results
-        ]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=pd.DataFrame(results_table),
-                    metadata=ResultTableMetadata(
-                        title="Training-Test Degradation Test"
-                    ),
-                )
-            ]
-        )
-
-    def run(self):
-        y_train_true = self.inputs.datasets[0].y
-        y_train_pred = self.inputs.datasets[0].y_pred(self.inputs.model)
-        y_train_true = y_train_true.astype(y_train_pred.dtype)
-
-        y_test_true = self.inputs.datasets[1].y
-        y_test_pred = self.inputs.datasets[1].y_pred(self.inputs.model)
-        y_test_true = y_test_true.astype(y_test_pred.dtype)
-
-        report_train = metrics.classification_report(
-            y_train_true, y_train_pred, output_dict=True, zero_division=0
-        )
-        report_train["roc_auc"] = multiclass_roc_auc_score(y_train_true, y_train_pred)
-
-        report_test = metrics.classification_report(
-            y_test_true, y_test_pred, output_dict=True, zero_division=0
-        )
-        report_test["roc_auc"] = multiclass_roc_auc_score(y_test_true, y_test_pred)
-
-        classes = {str(i) for i in unique(y_train_true)}
-
-        test_results = []
-        for class_name in classes:
-            for metric_name in ["precision", "recall", "f1-score"]:
-                train_score = report_train[class_name][metric_name]
-                test_score = report_test[class_name][metric_name]
-
-                # If training score is 0, degradation is assumed to be 100%
-                if train_score == 0:
-                    degradation = 1.0
-                else:
-                    degradation = (train_score - test_score) / train_score
-
-                passed = degradation < self.params["max_threshold"]
-                test_results.append(
-                    ThresholdTestResult(
-                        test_name=metric_name,
-                        passed=passed,
-                        values={
-                            "class": class_name,
-                            "test_score": test_score,
-                            "train_score": train_score,
-                            "degradation": degradation,
-                        },
-                    )
-                )
-        return self.cache_results(
-            test_results, passed=all(r.passed for r in test_results)
-        )
+    ds1_report = classification_report(
+        y_true=datasets[0].y,
+        y_pred=datasets[0].y_pred(model),
+        output_dict=True,
+        zero_division=0,
+    )
+    ds2_report = classification_report(
+        y_true=datasets[1].y,
+        y_pred=datasets[1].y_pred(model),
+        output_dict=True,
+        zero_division=0,
+    )
+
+    table = []
+
+    for class_name in {str(i) for i in unique(datasets[0].y)}:
+        for metric_name in ["precision", "recall", "f1-score"]:
+            ds1_score = ds1_report[class_name][metric_name]
+            ds2_score = ds2_report[class_name][metric_name]
+
+            # If training score is 0, degradation is assumed to be 100%
+            degradation = 1.0 if ds1_score == 0 else (ds1_score - ds2_score) / ds1_score
+            passed = degradation < max_threshold
+
+            table.append(
+                {
+                    "Class": class_name,
+                    "Metric": metric_name.title(),
+                    f"{datasets[0].input_id} Score": ds1_score,
+                    f"{datasets[1].input_id} Score": ds2_score,
+                    "Degradation (%)": degradation * 100,
+                    "Pass/Fail": "Pass" if passed else "Fail",
+                }
+            )
+
+    return table, all(row["Pass/Fail"] == "Pass" for row in table)

validmind/tests/model_validation/sklearn/VMeasure.py

@@ -2,15 +2,15 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 from sklearn import metrics
 
-from .ClusterPerformance import ClusterPerformance
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class VMeasure(ClusterPerformance):
+@tags("sklearn", "model_performance")
+@tasks("clustering")
+def VMeasure(dataset: VMDataset, model: VMModel):
     """
     Evaluates homogeneity and completeness of a clustering model using the V Measure Score.
 
@@ -48,14 +48,11 @@ class VMeasure(ClusterPerformance):
     the other. The V Measure Score does not provide flexibility in assigning different weights to homogeneity and
     completeness.
     """
-
-    name = "v_measure_score"
-    required_inputs = ["model", "dataset"]
-    tasks = ["clustering"]
-    tags = [
-        "sklearn",
-        "model_performance",
+    return [
+        {
+            "V Measure": metrics.v_measure_score(
+                labels_true=dataset.y,
+                labels_pred=dataset.y_pred(model),
+            )
+        }
     ]
-
-    def metric_info(self):
-        return {"V Measure": metrics.v_measure_score}