validmind 2.5.25__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/model_validation/FeaturesAUC.py

@@ -2,22 +2,22 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 from sklearn.metrics import roc_auc_score
 
+from validmind import tags, tasks
 from validmind.errors import SkipTestError
 from validmind.logging import get_logger
-from validmind.vm_models import Figure, Metric
+from validmind.vm_models import VMDataset
 
 logger = get_logger(__name__)
 
 
-@dataclass
-class FeaturesAUC(Metric):
+@tags("feature_importance", "AUC", "visualization")
+@tasks("classification")
+def FeaturesAUC(dataset: VMDataset, fontsize: int = 12, figure_height: int = 500):
     """
     Evaluates the discriminatory power of each individual feature within a binary classification model by calculating
     the Area Under the Curve (AUC) for each feature separately.
@@ -57,73 +57,42 @@ class FeaturesAUC(Metric):
     - This metric is applicable only to binary classification tasks and cannot be directly extended to multiclass
     classification or regression without modifications.
     """
-
-    name = "features_auc"
-    required_inputs = ["model", "dataset"]
-    default_params = {
-        "fontsize": 12,
-        "figure_height": 500,
-    }
-    tasks = ["classification"]
-    tags = [
-        "feature_importance",
-        "AUC",
-        "visualization",
-    ]
-
-    def run(self):
-        dataset = self.inputs.dataset
-        x = dataset.x_df()
-        y = dataset.y_df()
-        n_targets = dataset.df[dataset.target_column].nunique()
-
-        if n_targets != 2:
-            raise SkipTestError("FeaturesAUC metric requires a binary target variable.")
-
-        aucs = pd.DataFrame(index=x.columns, columns=["AUC"])
-
-        for column in x.columns:
-            feature_values = x[column]
-            if feature_values.nunique() > 1:
-                auc_score = roc_auc_score(y, feature_values)
-                aucs.loc[column, "AUC"] = auc_score
-            else:
-                aucs.loc[
-                    column, "AUC"
-                ] = np.nan  # Not enough unique values to calculate AUC
-
-        # Sorting the AUC scores in descending order
-        sorted_indices = aucs["AUC"].dropna().sort_values(ascending=False).index
-
-        # Plotting the results
-        fig = go.Figure()
-        fig.add_trace(
-            go.Bar(
-                y=[column for column in sorted_indices],
-                x=[aucs.loc[column, "AUC"] for column in sorted_indices],
-                orientation="h",
-            )
-        )
-        fig.update_layout(
-            title_text="Feature AUC Scores",
-            yaxis=dict(
-                tickmode="linear",
-                dtick=1,
-                tickfont=dict(size=self.params["fontsize"]),
-                title="Features",
-                autorange="reversed",  # Ensure that the highest AUC is at the top
-            ),
-            xaxis=dict(title="AUC"),
-            height=self.params["figure_height"],
-        )
-
-        return self.cache_results(
-            metric_value=aucs.to_dict(),
-            figures=[
-                Figure(
-                    for_object=self,
-                    key="features_auc",
-                    figure=fig,
-                ),
-            ],
+    if len(np.unique(dataset.y)) != 2:
+        raise SkipTestError("FeaturesAUC metric requires a binary target variable.")
+
+    aucs = pd.DataFrame(index=dataset.feature_columns, columns=["AUC"])
+
+    for column in dataset.feature_columns:
+        feature_values = dataset.df[column]
+        if feature_values.nunique() > 1 and pd.api.types.is_numeric_dtype(
+            feature_values
+        ):
+            aucs.loc[column, "AUC"] = roc_auc_score(dataset.y, feature_values)
+        else:
+            # Not enough unique values to calculate AUC
+            aucs.loc[column, "AUC"] = np.nan
+
+    sorted_indices = aucs["AUC"].dropna().sort_values(ascending=False).index
+
+    fig = go.Figure()
+    fig.add_trace(
+        go.Bar(
+            y=[column for column in sorted_indices],
+            x=[aucs.loc[column, "AUC"] for column in sorted_indices],
+            orientation="h",
         )
+    )
+    fig.update_layout(
+        title_text="Feature AUC Scores",
+        yaxis=dict(
+            tickmode="linear",
+            dtick=1,
+            tickfont=dict(size=fontsize),
+            title="Features",
+            autorange="reversed",  # Ensure that the highest AUC is at the top
+        ),
+        xaxis=dict(title="AUC"),
+        height=figure_height,
+    )
+
+    return fig

validmind/tests/model_validation/MeteorScore.py

@@ -7,6 +7,7 @@ import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import tags, tasks
+from validmind.tests.utils import validate_prediction
 
 
 @tags("nlp", "text_data", "visualization")
@@ -65,6 +66,8 @@ def MeteorScore(dataset, model):
     y_true = dataset.y
     y_pred = dataset.y_pred(model)
 
+    validate_prediction(y_true, y_pred)
+
     # Load the METEOR evaluation metric
     meteor = evaluate.load("meteor")
 

validmind/tests/model_validation/RegardScore.py

@@ -7,6 +7,7 @@ import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import tags, tasks
+from validmind.tests.utils import validate_prediction
 
 
 @tags("nlp", "text_data", "visualization")
@@ -56,8 +57,11 @@ def RegardScore(dataset, model):
     y_true = dataset.y
     y_pred = dataset.y_pred(model)
 
+    # Ensure equal lengths and get truncated data if necessary
+    y_true, y_pred = validate_prediction(y_true, y_pred)
+
     # Load the regard evaluation metric
-    regard_tool = evaluate.load("regard")
+    regard_tool = evaluate.load("regard", module_type="measurement")
 
     # Function to calculate regard scores
     def compute_regard_scores(texts):

validmind/tests/model_validation/RegressionResidualsPlot.py

@@ -2,17 +2,17 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-
 import numpy as np
 import plotly.figure_factory as ff
 import plotly.graph_objects as go
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
-@dataclass
-class RegressionResidualsPlot(Metric):
+@tags("model_performance", "visualization")
+@tasks("regression")
+def RegressionResidualsPlot(model: VMModel, dataset: VMDataset, bin_size: float = 0.1):
     """
     Evaluates regression model performance using residual distribution and actual vs. predicted plots.
 
@@ -54,75 +54,54 @@ class RegressionResidualsPlot(Metric):
     - Does not summarize model performance into a single quantifiable metric, which might be needed for comparative or
     summary analyses.
     """
-
-    name = "regression_residuals_plot"
-    required_inputs = ["model", "dataset"]
-    tasks = ["regression"]
-    tags = ["model_performance"]
-    default_params = {"bin_size": 0.1}
-
-    def run(self):
-        y_true = self.inputs.dataset.y
-        y_pred = self.inputs.dataset.y_pred(self.inputs.model)
-        # Calculate residuals
-        residuals = y_true.flatten() - y_pred.flatten()
-        # Create residuals plot
-        hist_data = [residuals]
-        group_labels = ["Residuals"]  # Names of the dataset
-        bin_size = self.params["bin_size"]
-        fig = ff.create_distplot(
-            hist_data, group_labels, bin_size=[bin_size], show_hist=True, show_rug=False
-        )
-        fig.update_layout(
-            title="Distribution of Residuals",
-            xaxis_title="Residuals",
-            yaxis_title="Density",
-        )
-        figures = [
-            Figure(
-                for_object=self,
-                key=self.key,
-                figure=fig,
-            )
-        ]
-        # Create a scatter plot of actual vs predicted values
-        scatter = go.Scatter(
-            x=y_true.flatten(),
-            y=y_pred.flatten(),
-            mode="markers",
-            name="True vs Predicted",
-            marker=dict(color="blue", opacity=0.5),
-        )
-
-        # Line of perfect prediction
-        max_val = np.nanmax([np.nanmax(y_true), np.nanmax(y_pred)])
-        min_val = np.nanmin([np.nanmin(y_true), np.nanmin(y_pred)])
-        line = go.Scatter(
-            x=[min_val, max_val],
-            y=[min_val, max_val],
-            mode="lines",
-            name="Perfect Fit",
-            line=dict(color="red", dash="dash"),
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    figures = []
+
+    # Residuals plot
+    fig = ff.create_distplot(
+        hist_data=[y_true.flatten() - y_pred.flatten()],
+        group_labels=["Residuals"],
+        bin_size=[bin_size],
+        show_hist=True,
+        show_rug=False,
+    )
+    fig.update_layout(
+        title="Distribution of Residuals",
+        xaxis_title="Residuals",
+        yaxis_title="Density",
+    )
+    figures.append(fig)
+
+    # True vs Predicted w/ perfect fit line plot
+    max_val = np.nanmax([np.nanmax(y_true), np.nanmax(y_pred)])
+    min_val = np.nanmin([np.nanmin(y_true), np.nanmin(y_pred)])
+    figures.append(
+        go.Figure(
+            data=[
+                go.Scatter(
+                    x=y_true.flatten(),
+                    y=y_pred.flatten(),
+                    mode="markers",
+                    name="True vs Predicted",
+                    marker=dict(color="blue", opacity=0.5),
+                ),
+                go.Scatter(
+                    x=[min_val, max_val],
+                    y=[min_val, max_val],
+                    mode="lines",
+                    name="Perfect Fit",
+                    line=dict(color="red", dash="dash"),
+                ),
+            ],
+            layout=go.Layout(
+                title="True vs. Predicted Values",
+                xaxis_title="True Values",
+                yaxis_title="Predicted Values",
+                showlegend=True,
+            ),
         )
+    )
 
-        # Layout settings
-        layout = go.Layout(
-            title="True vs. Predicted Values",
-            xaxis_title="True Values",
-            yaxis_title="Predicted Values",
-            showlegend=True,
-        )
-
-        fig = go.Figure(data=[scatter, line], layout=layout)
-
-        figures.append(
-            Figure(
-                for_object=self,
-                key=self.key,
-                figure=fig,
-            )
-        )
-
-        return self.cache_results(
-            figures=figures,
-        )
+    return tuple(figures)

validmind/tests/model_validation/embeddings/ClusterDistribution.py

@@ -5,10 +5,13 @@
 import plotly.express as px
 from sklearn.cluster import KMeans
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
-class ClusterDistribution(Metric):
+@tags("llm", "text_data", "embeddings", "visualization")
+@tasks("feature_extraction")
+def ClusterDistribution(model: VMModel, dataset: VMDataset, num_clusters: int = 5):
     """
     Assesses the distribution of text embeddings across clusters produced by a model using KMeans clustering.
 
@@ -49,34 +52,8 @@ class ClusterDistribution(Metric):
     - Uses the KMeans clustering algorithm, which assumes that clusters are convex and isotropic, and may not work as
     intended if the true clusters in the data are not of this shape.
     """
-
-    name = "Text Embeddings Cluster Distribution"
-    required_inputs = ["model", "dataset"]
-    default_params = {
-        "num_clusters": 5,
-    }
-    tasks = ["feature_extraction"]
-    tags = ["llm", "text_data", "embeddings", "visualization"]
-
-    def run(self):
-        # run kmeans clustering on embeddings
-        kmeans = KMeans(n_clusters=self.params["num_clusters"]).fit(
-            self.inputs.dataset.y_pred(self.inputs.model)
-        )
-
-        # plot the distribution
-        fig = px.histogram(
-            kmeans.labels_,
-            nbins=self.params["num_clusters"],
-            title="Embeddings Cluster Distribution",
-        )
-
-        return self.cache_results(
-            figures=[
-                Figure(
-                    for_object=self,
-                    key=self.key,
-                    figure=fig,
-                )
-            ],
-        )
+    return px.histogram(
+        KMeans(n_clusters=num_clusters).fit(dataset.y_pred(model)).labels_,
+        nbins=num_clusters,
+        title="Embeddings Cluster Distribution",
+    )

validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py

@@ -5,10 +5,13 @@
 import plotly.express as px
 from sklearn.metrics.pairwise import cosine_similarity
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
-class CosineSimilarityDistribution(Metric):
+@tags("llm", "text_data", "embeddings", "visualization")
+@tasks("feature_extraction")
+def CosineSimilarityDistribution(dataset: VMDataset, model: VMModel):
     """
     Assesses the similarity between predicted text embeddings from a model using a Cosine Similarity distribution
     histogram.
@@ -49,30 +52,9 @@ class CosineSimilarityDistribution(Metric):
     - The output is sensitive to the choice of bin number for the histogram. Different bin numbers could give a
     slightly altered perspective on the distribution of cosine similarity.
     """
-
-    name = "Text Embeddings Cosine Similarity Distribution"
-    required_inputs = ["model", "dataset"]
-    tasks = ["feature_extraction"]
-    tags = ["llm", "text_data", "embeddings", "visualization"]
-
-    def run(self):
-        # Compute cosine similarity
-        similarities = cosine_similarity(self.inputs.dataset.y_pred(self.inputs.model))
-
-        # plot the distribution
-        fig = px.histogram(
-            x=similarities.flatten(),
-            nbins=100,
-            title="Cosine Similarity Distribution",
-            labels={"x": "Cosine Similarity"},
-        )
-
-        return self.cache_results(
-            figures=[
-                Figure(
-                    for_object=self,
-                    key=self.key,
-                    figure=fig,
-                )
-            ],
-        )
+    return px.histogram(
+        x=cosine_similarity(dataset.y_pred(model)).flatten(),
+        nbins=100,
+        title="Cosine Similarity Distribution",
+        labels={"x": "Cosine Similarity"},
+    )

validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py

@@ -5,10 +5,13 @@
 import numpy as np
 import plotly.express as px
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
 
 
-class DescriptiveAnalytics(Metric):
+@tags("llm", "text_data", "embeddings", "visualization")
+@tasks("feature_extraction")
+def DescriptiveAnalytics(dataset: VMDataset, model: VMModel):
     """
     Evaluates statistical properties of text embeddings in an ML model via mean, median, and standard deviation
     histograms.
@@ -52,32 +55,17 @@ class DescriptiveAnalytics(Metric):
     - While it displays valuable information about the central tendency and spread of data, it does not provide
     information about correlations between different embedding dimensions.
     """
-
-    name = "Descriptive Analytics for Text Embeddings Models"
-    required_inputs = ["model", "dataset"]
-    tasks = ["feature_extraction"]
-    tags = ["llm", "text_data", "embeddings", "visualization"]
-
-    def run(self):
-        # Assuming y_pred returns a 2D array of embeddings [samples, features]
-        preds = self.inputs.dataset.y_pred(self.inputs.model)
-
-        # Calculate statistics across the embedding dimensions, not across all embeddings
-        means = np.mean(preds, axis=0)  # Mean of each feature across all samples
-        medians = np.median(preds, axis=0)  # Median of each feature across all samples
-        stds = np.std(preds, axis=0)  # Std. dev. of each feature across all samples
-
-        # Plot histograms of the calculated statistics
-        mean_fig = px.histogram(x=means, title="Distribution of Embedding Means")
-        median_fig = px.histogram(x=medians, title="Distribution of Embedding Medians")
-        std_fig = px.histogram(
-            x=stds, title="Distribution of Embedding Standard Deviations"
-        )
-
-        return self.cache_results(
-            figures=[
-                Figure(for_object=self, key=f"{self.key}_mean", figure=mean_fig),
-                Figure(for_object=self, key=f"{self.key}_median", figure=median_fig),
-                Figure(for_object=self, key=f"{self.key}_std", figure=std_fig),
-            ],
-        )
+    return (
+        px.histogram(
+            x=np.mean(dataset.y_pred(model), axis=0),
+            title="Distribution of Embedding Means",
+        ),
+        px.histogram(
+            x=np.median(dataset.y_pred(model), axis=0),
+            title="Distribution of Embedding Medians",
+        ),
+        px.histogram(
+            x=np.std(dataset.y_pred(model), axis=0),
+            title="Distribution of Embedding Standard Deviations",
+        ),
+    )

validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py

@@ -2,13 +2,26 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Union
+
 import plotly.express as px
 from sklearn.manifold import TSNE
 
-from validmind.vm_models import Figure, Metric
+from validmind import tags, tasks
+from validmind.logging import get_logger
+from validmind.vm_models import VMDataset, VMModel
+
+logger = get_logger(__name__)
 
 
-class EmbeddingsVisualization2D(Metric):
+@tags("llm", "text_data", "embeddings", "visualization")
+@tasks("feature_extraction")
+def EmbeddingsVisualization2D(
+    model: VMModel,
+    dataset: VMDataset,
+    cluster_column: Union[str, None] = None,
+    perplexity: int = 30,
+):
     """
     Visualizes 2D representation of text embeddings generated by a model using t-SNE technique.
 
@@ -50,52 +63,30 @@ class EmbeddingsVisualization2D(Metric):
     - It is designed for visual exploration and not for downstream tasks; that is, the 2D embeddings generated should
     not be directly used for further training or analysis.
     """
+    y_pred = dataset.y_pred(model)
+
+    num_samples = len(y_pred)
+    perplexity = perplexity if perplexity < num_samples else num_samples - 1
+
+    reduced_embeddings = TSNE(
+        n_components=2,
+        perplexity=perplexity,
+    ).fit_transform(y_pred)
 
-    name = "2D Visualization of Text Embeddings"
-    required_inputs = ["model", "dataset"]
-    default_params = {
-        "cluster_column": None,
-        "perplexity": 30,
+    if not cluster_column and len(dataset.feature_columns_categorical) == 1:
+        cluster_column = dataset.feature_columns_categorical[0]
+    else:
+        logger.warning("Cannot color code embeddings without a 'cluster_column' param.")
+
+    scatter_kwargs = {
+        "x": reduced_embeddings[:, 0],
+        "y": reduced_embeddings[:, 1],
+        "title": "2D Visualization of Text Embeddings",
     }
-    tasks = ["feature_extraction"]
-    tags = ["llm", "text_data", "embeddings", "visualization"]
-
-    def run(self):
-        cluster_column = self.params.get("cluster_column")
-
-        if cluster_column is None:
-            raise ValueError(
-                "The `cluster_column` parameter must be provided to the EmbeddingsVisualization2D test."
-            )
-
-        # use TSNE to reduce dimensionality of embeddings
-        num_samples = len(self.inputs.dataset.y_pred(self.inputs.model))
-
-        if self.params["perplexity"] >= num_samples:
-            perplexity = num_samples - 1
-        else:
-            perplexity = self.params["perplexity"]
-
-        reduced_embeddings = TSNE(
-            n_components=2,
-            perplexity=perplexity,
-        ).fit_transform(self.inputs.model.y_test_predict.values)
-
-        # create a scatter plot from the reduced embeddings
-        fig = px.scatter(
-            x=reduced_embeddings[:, 0],
-            y=reduced_embeddings[:, 1],
-            color=self.inputs.dataset.df[cluster_column],
-            title="2D Visualization of Text Embeddings",
-        )
-        fig.update_layout(width=500, height=500)
-
-        return self.cache_results(
-            figures=[
-                Figure(
-                    for_object=self,
-                    key=self.key,
-                    figure=fig,
-                )
-            ],
-        )
+    if cluster_column:
+        scatter_kwargs["color"] = dataset.df[cluster_column]
+
+    fig = px.scatter(**scatter_kwargs)
+    fig.update_layout(width=500, height=500)
+
+    return fig