validmind 2.3.5__py3-none-any.whl → 2.4.1__py3-none-any.whl

validmind/tests/metadata.py

@@ -0,0 +1,59 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+
+from validmind.utils import format_dataframe
+
+from .load import list_tests
+
+
+def list_tags():
+    """
+    List unique tags from all test classes.
+    """
+
+    unique_tags = set()
+
+    for test in list_tests(__as_class=True):
+        unique_tags.update(test.tags)
+
+    return list(unique_tags)
+
+
+def list_tasks_and_tags():
+    """
+    List all task types and their associated tags, with one row per task type and
+    all tags for a task type in one row.
+
+    Returns:
+        pandas.DataFrame: A DataFrame with 'Task Type' and concatenated 'Tags'.
+    """
+    task_tags_dict = {}
+
+    for test in list_tests(__as_class=True):
+        for task in test.tasks:
+            task_tags_dict.setdefault(task, set()).update(test.tags)
+
+    return format_dataframe(
+        pd.DataFrame(
+            [
+                {"Task": task, "Tags": ", ".join(tags)}
+                for task, tags in task_tags_dict.items()
+            ]
+        )
+    )
+
+
+def list_tasks():
+    """
+    List unique tasks from all test classes.
+    """
+
+    unique_tasks = set()
+
+    for test in list_tests(__as_class=True):
+        unique_tasks.update(test.tasks)
+
+    return list(unique_tasks)

validmind/tests/model_validation/ClusterSizeDistribution.py

@@ -51,13 +51,11 @@ class ClusterSizeDistribution(Metric):
 
     name = "cluster_size_distribution"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def run(self):
         y_true_train = self.inputs.dataset.y

validmind/tests/model_validation/FeaturesAUC.py

@@ -45,14 +45,12 @@ class FeaturesAUC(Metric):
         "fontsize": 12,
         "figure_height": 500,
     }
-    metadata = {
-        "task_types": ["classification"],
-        "tags": [
-            "feature_importance",
-            "AUC",
-            "visualization",
-        ],
-    }
+    tasks = ["classification"]
+    tags = [
+        "feature_importance",
+        "AUC",
+        "visualization",
+    ]
 
     def run(self):
         dataset = self.inputs.dataset

validmind/tests/model_validation/ModelMetadata.py

@@ -53,15 +53,14 @@ class ModelMetadata(Metric):
 
     name = "model_metadata"
     required_inputs = ["model"]
-    metadata = {
-        "task_types": [
-            "classification",
-            "regression",
-            "text_classification",
-            "text_summarization",
-        ],
-        "tags": ["model_metadata"],
-    }
+    tasks = [
+        "classification",
+        "regression",
+        "text_classification",
+        "text_summarization",
+    ]
+
+    tags = ["model_metadata"]
 
     column_labels = {
         "architecture": "Modeling Technique",

validmind/tests/model_validation/RegressionResidualsPlot.py

@@ -52,12 +52,8 @@ class RegressionResidualsPlot(Metric):
 
     name = "regression_residuals_plot"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["regression"],
-        "tags": [
-            "model_performance",
-        ],
-    }
+    tasks = ["regression"]
+    tags = ["model_performance"]
     default_params = {"bin_size": 0.1}
 
     def run(self):

validmind/tests/model_validation/embeddings/ClusterDistribution.py

@@ -51,10 +51,8 @@ class ClusterDistribution(Metric):
     default_params = {
         "num_clusters": 5,
     }
-    metadata = {
-        "task_types": ["feature_extraction"],
-        "tags": ["llm", "text_data", "text_embeddings", "visualization"],
-    }
+    tasks = ["feature_extraction"]
+    tags = ["llm", "text_data", "text_embeddings", "visualization"]
 
     def run(self):
         # run kmeans clustering on embeddings

validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py

@@ -50,10 +50,8 @@ class CosineSimilarityDistribution(Metric):
 
     name = "Text Embeddings Cosine Similarity Distribution"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["feature_extraction"],
-        "tags": ["llm", "text_data", "text_embeddings", "visualization"],
-    }
+    tasks = ["feature_extraction"]
+    tags = ["llm", "text_data", "text_embeddings", "visualization"]
 
     def run(self):
         # Compute cosine similarity

validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py

@@ -53,10 +53,8 @@ class DescriptiveAnalytics(Metric):
 
     name = "Descriptive Analytics for Text Embeddings Models"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["feature_extraction"],
-        "tags": ["llm", "text_data", "text_embeddings", "visualization"],
-    }
+    tasks = ["feature_extraction"]
+    tags = ["llm", "text_data", "text_embeddings", "visualization"]
 
     def run(self):
         # Assuming y_pred returns a 2D array of embeddings [samples, features]

validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py

@@ -53,10 +53,8 @@ class EmbeddingsVisualization2D(Metric):
         "cluster_column": None,
         "perplexity": 30,
     }
-    metadata = {
-        "task_types": ["feature_extraction"],
-        "tags": ["llm", "text_data", "text_embeddings", "visualization"],
-    }
+    tasks = ["feature_extraction"]
+    tags = ["llm", "text_data", "text_embeddings", "visualization"]
 
     def run(self):
         cluster_column = self.params.get("cluster_column")

validmind/tests/model_validation/embeddings/StabilityAnalysis.py

@@ -29,10 +29,8 @@ class StabilityAnalysis(ThresholdTest):
     default_params = {
         "mean_similarity_threshold": 0.7,
     }
-    metadata = {
-        "task_types": ["feature_extraction"],
-        "tags": ["llm", "text_data", "text_embeddings", "visualization"],
-    }
+    tasks = ["feature_extraction"]
+    tags = ["llm", "text_data", "text_embeddings", "visualization"]
 
     @abstractmethod
     def perturb_data(self, data: str) -> str:

validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py

@@ -48,13 +48,11 @@ class AdjustedMutualInformation(ClusterPerformance):
 
     name = "adjusted_mutual_information"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def metric_info(self):
         return {"Adjusted Mutual Information": metrics.adjusted_mutual_info_score}

validmind/tests/model_validation/sklearn/AdjustedRandIndex.py

@@ -47,13 +47,11 @@ class AdjustedRandIndex(ClusterPerformance):
 
     name = "adjusted_rand_index"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def metric_info(self):
         return {"Adjusted Rand Index": metrics.adjusted_rand_score}

validmind/tests/model_validation/sklearn/ClassifierPerformance.py

@@ -58,15 +58,13 @@ class ClassifierPerformance(Metric):
 
     name = "classifier_performance"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "model_performance",
-        ],
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "model_performance",
+    ]
 
     def summary(self, metric_value: dict):
         """

validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py

@@ -57,13 +57,11 @@ class ClusterCosineSimilarity(Metric):
 
     name = "cluster_cosine_similarity"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def run(self):
         y_true_train = self.inputs.dataset.y

validmind/tests/model_validation/sklearn/ClusterPerformance.py

@@ -51,13 +51,11 @@ class ClusterPerformance(Metric):
 
     name = "cluster_performance_metrics"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def cluser_performance_metrics(
         self, y_true_train, y_pred_train, y_true_test, y_pred_test, samples, metric_info

validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py

@@ -61,13 +61,8 @@ class ClusterPerformanceMetrics(ClusterPerformance):
 
     name = "homogeneity_score"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = ["sklearn", "model_performance"]
     default_metrics = {
         "Homogeneity Score": metrics.homogeneity_score,
         "Completeness Score": metrics.completeness_score,

validmind/tests/model_validation/sklearn/CompletenessScore.py

@@ -44,13 +44,11 @@ class CompletenessScore(ClusterPerformance):
 
     name = "homogeneity_score"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def metric_info(self):
         return {"Completeness Score": metrics.completeness_score}

validmind/tests/model_validation/sklearn/ConfusionMatrix.py

@@ -55,16 +55,14 @@ class ConfusionMatrix(Metric):
 
     name = "confusion_matrix"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "model_performance",
-            "visualization",
-        ],
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "model_performance",
+        "visualization",
+    ]
 
     def run(self):
         y_true = self.inputs.dataset.y
@@ -113,6 +111,17 @@ class ConfusionMatrix(Metric):
             height=600,
         )
 
+        # Add an annotation at the bottom of the heatmap
+        fig.add_annotation(
+            x=0.5,
+            y=-0.1,
+            xref="paper",
+            yref="paper",
+            text=f"Confusion Matrix for {self.inputs.model.input_id} on {self.inputs.dataset.input_id}",
+            showarrow=False,
+            font=dict(size=14),
+        )
+
         return self.cache_results(
             metric_value={
                 "confusion_matrix": cm,

validmind/tests/model_validation/sklearn/FowlkesMallowsScore.py

@@ -55,13 +55,11 @@ class FowlkesMallowsScore(ClusterPerformance):
 
     name = "fowlkes_mallows_score"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def metric_info(self):
         return {"Fowlkes-Mallows score": metrics.fowlkes_mallows_score}

validmind/tests/model_validation/sklearn/HomogeneityScore.py

@@ -46,13 +46,11 @@ class HomogeneityScore(ClusterPerformance):
 
     name = "homogeneity_score"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["clustering"]
+    tags = [
+        "sklearn",
+        "model_performance",
+    ]
 
     def metric_info(self):
         return {"Homogeneity Score": metrics.homogeneity_score}

validmind/tests/model_validation/sklearn/HyperParametersTuning.py

@@ -52,13 +52,8 @@ class HyperParametersTuning(Metric):
 
     name = "hyper_parameters_tuning"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["classification", "clustering"],
-        "tags": [
-            "sklearn",
-            "model_performance",
-        ],
-    }
+    tasks = ["classification", "clustering"]
+    tags = ["sklearn", "model_performance"]
     default_params = {"param_grid": None, "scoring": None}
 
     def run(self):

validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py

@@ -60,13 +60,10 @@ class KMeansClustersOptimization(Metric):
 
     name = "clusters_optimize_elbow_method"
     required_inputs = ["model", "dataset"]
-    metadata = {
-        "task_types": ["clustering"],
-        "tags": ["sklearn", "model_performance", "kmeans"],
-    }
-    default_params = {
-        "n_clusters": None,
-    }
+    tasks = ["clustering"]
+    tags = ["sklearn", "model_performance", "kmeans"]
+
+    default_params = {"n_clusters": None}
 
     def run(self):
         n_clusters = self.params["n_clusters"]

validmind/tests/model_validation/sklearn/MinimumAccuracy.py

@@ -59,15 +59,13 @@ class MinimumAccuracy(ThresholdTest):
     name = "accuracy_score"
     required_inputs = ["model", "dataset"]
     default_params = {"min_threshold": 0.7}
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "model_performance",
-        ],
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "model_performance",
+    ]
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         """

validmind/tests/model_validation/sklearn/MinimumF1Score.py

@@ -62,15 +62,13 @@ class MinimumF1Score(ThresholdTest):
     name = "f1_score"
     required_inputs = ["model", "dataset"]
     default_params = {"min_threshold": 0.5}
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "model_performance",
-        ],
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "model_performance",
+    ]
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         """

validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py

@@ -59,15 +59,13 @@ class MinimumROCAUCScore(ThresholdTest):
     name = "roc_auc_score"
     required_inputs = ["model", "dataset"]
     default_params = {"min_threshold": 0.5}
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "model_performance",
-        ],
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "model_performance",
+    ]
 
     def summary(self, results: List[ThresholdTestResult], all_passed: bool):
         """

validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.py

@@ -53,16 +53,14 @@ class ModelsPerformanceComparison(ClassifierPerformance):
 
     name = "models_performance_comparison"
     required_inputs = ["dataset", "models"]
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "model_performance",
-            "model_comparison",
-        ],
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "model_performance",
+        "model_comparison",
+    ]
 
     def summary(self, metric_value: dict):
         """

validmind/tests/model_validation/sklearn/OverfitDiagnosis.py

@@ -67,15 +67,13 @@ class OverfitDiagnosis(ThresholdTest):
     name = "overfit_regions"
     required_inputs = ["model", "datasets"]
     default_params = {"features_columns": None, "cut_off_percentage": 4}
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "model_diagnosis",
-        ],
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "model_diagnosis",
+    ]
 
     default_metrics = {
         "accuracy": metrics.accuracy_score,

validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py

@@ -56,16 +56,14 @@ class PermutationFeatureImportance(Metric):
         "fontsize": None,
         "figure_height": 1000,
     }
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "feature_importance",
-            "visualization",
-        ],
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "feature_importance",
+        "visualization",
+    ]
 
     def run(self):
         x = self.inputs.dataset.x_df()

validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py

@@ -73,15 +73,13 @@ class PopulationStabilityIndex(Metric):
 
     name = "psi"
     required_inputs = ["model", "datasets"]
-    metadata = {
-        "task_types": ["classification", "text_classification"],
-        "tags": [
-            "sklearn",
-            "binary_classification",
-            "multiclass_classification",
-            "model_performance",
-        ],
-    }
+    tasks = ["classification", "text_classification"]
+    tags = [
+        "sklearn",
+        "binary_classification",
+        "multiclass_classification",
+        "model_performance",
+    ]
     default_params = {
         "num_bins": 10,
         "mode": "fixed",