validmind 2.7.5__py3-none-any.whl → 2.7.7__py3-none-any.whl

validmind/tests/model_validation/ToxicityScore.py

@@ -6,7 +6,7 @@ import evaluate
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("nlp", "text_data", "visualization")
@@ -139,4 +139,12 @@ def ToxicityScore(dataset, model):
         ]
     ]
 
-    return (result_df, *tuple(figures))
+    return (
+        result_df,
+        *tuple(figures),
+        RawData(
+            input_toxicity_df=input_df,
+            true_toxicity_df=true_df,
+            pred_toxicity_df=pred_df,
+        ),
+    )

validmind/tests/model_validation/embeddings/ClusterDistribution.py

@@ -5,7 +5,7 @@
 import plotly.express as px
 from sklearn.cluster import KMeans
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -52,8 +52,14 @@ def ClusterDistribution(model: VMModel, dataset: VMDataset, num_clusters: int =
     - Uses the KMeans clustering algorithm, which assumes that clusters are convex and isotropic, and may not work as
     intended if the true clusters in the data are not of this shape.
     """
-    return px.histogram(
-        KMeans(n_clusters=num_clusters).fit(dataset.y_pred(model)).labels_,
+    embeddings = dataset.y_pred(model)
+    kmeans = KMeans(n_clusters=num_clusters).fit(embeddings)
+    labels = kmeans.labels_
+
+    fig = px.histogram(
+        labels,
         nbins=num_clusters,
         title="Embeddings Cluster Distribution",
     )
+
+    return fig, RawData(labels=labels)

validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py

@@ -9,7 +9,7 @@ import pandas as pd
 import plotly.express as px
 from sklearn.metrics.pairwise import cosine_similarity
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
@@ -63,6 +63,7 @@ def CosineSimilarityComparison(dataset, models):
     figures = []
     # Initialize a list to store data for the DataFrame
     all_stats = []
+    similarity_matrices = []
 
     # Generate all pairs of models for comparison
     for model_A, model_B in combinations(models, 2):
@@ -73,6 +74,15 @@ def CosineSimilarityComparison(dataset, models):
         similarity_matrix = cosine_similarity(embeddings_A, embeddings_B)
         similarities = similarity_matrix.flatten()
 
+        # store similarity matrix
+        similarity_matrices.append(
+            {
+                "model_A": model_A.input_id,
+                "model_B": model_B.input_id,
+                "similarity_matrix": similarity_matrix,
+            }
+        )
+
         # Generate statistics and add model combination as a column
         stats_data = {
             "Combination": f"{model_A.input_id} vs {model_B.input_id}",
@@ -100,4 +110,8 @@ def CosineSimilarityComparison(dataset, models):
     # Create a DataFrame from all collected statistics
     stats_df = pd.DataFrame(all_stats)
 
-    return (stats_df, *tuple(figures))
+    return (
+        *figures,
+        stats_df,
+        RawData(similarity_matrices=pd.DataFrame(similarity_matrices)),
+    )

validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py

@@ -5,7 +5,7 @@
 import plotly.express as px
 from sklearn.metrics.pairwise import cosine_similarity
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -52,9 +52,11 @@ def CosineSimilarityDistribution(dataset: VMDataset, model: VMModel):
     - The output is sensitive to the choice of bin number for the histogram. Different bin numbers could give a
     slightly altered perspective on the distribution of cosine similarity.
     """
+    similarity_scores = cosine_similarity(dataset.y_pred(model)).flatten()
+
     return px.histogram(
-        x=cosine_similarity(dataset.y_pred(model)).flatten(),
+        x=similarity_scores,
         nbins=100,
         title="Cosine Similarity Distribution",
         labels={"x": "Cosine Similarity"},
-    )
+    ), RawData(similarity_scores=similarity_scores)

validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py

@@ -6,7 +6,7 @@ import numpy as np
 import plotly.express as px
 from sklearn.metrics.pairwise import cosine_similarity
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
@@ -81,4 +81,4 @@ def CosineSimilarityHeatmap(
         yaxis_title=yaxis_title,
     )
 
-    return fig
+    return fig, RawData(similarity_matrix=similarity_matrix)

validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py

@@ -5,7 +5,7 @@
 import numpy as np
 import plotly.express as px
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -55,17 +55,27 @@ def DescriptiveAnalytics(dataset: VMDataset, model: VMModel):
     - While it displays valuable information about the central tendency and spread of data, it does not provide
     information about correlations between different embedding dimensions.
     """
+    y_pred = dataset.y_pred(model)
+    embedding_means = np.mean(y_pred, axis=0)
+    embedding_medians = np.median(y_pred, axis=0)
+    embedding_stds = np.std(y_pred, axis=0)
+
     return (
         px.histogram(
-            x=np.mean(dataset.y_pred(model), axis=0),
+            x=embedding_means,
             title="Distribution of Embedding Means",
         ),
         px.histogram(
-            x=np.median(dataset.y_pred(model), axis=0),
+            x=embedding_medians,
             title="Distribution of Embedding Medians",
         ),
         px.histogram(
-            x=np.std(dataset.y_pred(model), axis=0),
+            x=embedding_stds,
             title="Distribution of Embedding Standard Deviations",
         ),
+        RawData(
+            embedding_means=embedding_means,
+            embedding_medians=embedding_medians,
+            embedding_stds=embedding_stds,
+        ),
     )

validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py

@@ -7,7 +7,7 @@ from typing import Union
 import plotly.express as px
 from sklearn.manifold import TSNE
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
@@ -89,4 +89,4 @@ def EmbeddingsVisualization2D(
     fig = px.scatter(**scatter_kwargs)
     fig.update_layout(width=500, height=500)
 
-    return fig
+    return fig, RawData(tsne_embeddings=reduced_embeddings)

validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py

@@ -9,7 +9,7 @@ import pandas as pd
 import plotly.express as px
 from sklearn.metrics.pairwise import euclidean_distances
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
@@ -57,6 +57,8 @@ def EuclideanDistanceComparison(dataset, models):
     figures = []
     all_stats = []
 
+    distance_matrices = {}
+
     # Generate all pairs of models for comparison
     for model_A, model_B in combinations(models, 2):
         embeddings_A = np.stack(dataset.y_pred(model_A))
@@ -66,6 +68,15 @@ def EuclideanDistanceComparison(dataset, models):
         distance_matrix = euclidean_distances(embeddings_A, embeddings_B)
         distances = distance_matrix.flatten()
 
+        # Store raw distance matrix for each pair-wise comparison
+        distance_matrices.append(
+            {
+                "model_A": model_A.input_id,
+                "model_B": model_B.input_id,
+                "distance_matrix": distance_matrix,
+            }
+        )
+
         # Generate statistics and add model combination as a column
         stats_data = {
             "Combination": f"{model_A.input_id} vs {model_B.input_id}",
@@ -93,4 +104,7 @@ def EuclideanDistanceComparison(dataset, models):
     # Create a DataFrame from all collected statistics
     stats_df = pd.DataFrame(all_stats)
 
-    return (stats_df, *tuple(figures))
+    # Add raw data to return
+    raw_data = RawData(distance_matrices=pd.DataFrame(distance_matrices))
+
+    return (stats_df, *figures, raw_data)

validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py

@@ -6,7 +6,7 @@ import numpy as np
 import plotly.express as px
 from sklearn.metrics.pairwise import euclidean_distances
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
@@ -79,4 +79,4 @@ def EuclideanDistanceHeatmap(
         yaxis_title=yaxis_title,
     )
 
-    return fig
+    return fig, RawData(distance_matrix=distance_matrix)

validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py

@@ -10,7 +10,7 @@ import plotly.express as px
 from sklearn.decomposition import PCA
 from sklearn.preprocessing import StandardScaler
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
@@ -74,7 +74,7 @@ def PCAComponentsPairwisePlots(dataset, model, n_components=3):
     )
 
     # List to store each plot
-    plots = []
+    figures = []
 
     # Create plots for each pair of principal components
     for pc1, pc2 in itertools.combinations(range(1, n_components + 1), 2):
@@ -88,7 +88,6 @@ def PCAComponentsPairwisePlots(dataset, model, n_components=3):
                 f"PC{pc2}": f"Principal Component {pc2}",
             },
         )
-        plots.append(fig)
+        figures.append(fig)
 
-    # Return the list of plots as a tuple
-    return tuple(plots)
+    return (*figures, RawData(pca_results=pca_df))

validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py

@@ -5,7 +5,7 @@
 import re
 from typing import Dict
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 from .utils import create_stability_analysis_result
@@ -91,8 +91,10 @@ def StabilityAnalysisKeyword(
         perturb_data
     )
 
-    return create_stability_analysis_result(
+    raw_data, results = create_stability_analysis_result(
         dataset.y_pred(model),
         model.predict(perturbed_df),
         mean_similarity_threshold,
     )
+
+    return results, RawData(original_perturbed_similarity=raw_data)

validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py

@@ -5,7 +5,7 @@
 import random
 import string
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 from .utils import create_stability_analysis_result
@@ -145,8 +145,10 @@ def StabilityAnalysisRandomNoise(
         perturb_data
     )
 
-    return create_stability_analysis_result(
+    raw_data, result = create_stability_analysis_result(
         dataset.y_pred(model),
         model.predict(perturbed_df),
         mean_similarity_threshold,
     )
+
+    return result, RawData(original_perturbed_similarity=raw_data)

validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py

@@ -7,7 +7,7 @@ import random
 import nltk
 from nltk.corpus import wordnet as wn
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 from .utils import create_stability_analysis_result
@@ -101,8 +101,10 @@ def StabilityAnalysisSynonyms(
         perturb_data
     )
 
-    return create_stability_analysis_result(
+    raw_data, result = create_stability_analysis_result(
         dataset.y_pred(model),
         model.predict(perturbed_df),
         mean_similarity_threshold,
     )
+
+    return result, RawData(original_perturbed_similarity=raw_data)

validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py

@@ -4,7 +4,7 @@
 
 from transformers import MarianMTModel, MarianTokenizer
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
@@ -128,8 +128,10 @@ def StabilityAnalysisTranslation(
         perturb_data
     )
 
-    return create_stability_analysis_result(
+    raw_data, result = create_stability_analysis_result(
         dataset.y_pred(model),
         model.predict(perturbed_df),
         mean_similarity_threshold,
     )
+
+    return result, RawData(original_perturbed_similarity=raw_data)

validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py

@@ -10,7 +10,7 @@ import plotly.express as px
 from sklearn.manifold import TSNE
 from sklearn.preprocessing import StandardScaler
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization", "dimensionality_reduction", "embeddings")
@@ -80,7 +80,7 @@ def TSNEComponentsPairwisePlots(
     )
 
     # List to store each plot
-    plots = []
+    figures = []
 
     # Create plots for each pair of t-SNE components (if n_components > 1)
     if n_components > 1:
@@ -95,7 +95,7 @@ def TSNEComponentsPairwisePlots(
                     f"Component {comp2}": f"Component {comp2}",
                 },
             )
-            plots.append(fig)
+            figures.append(fig)
     else:
         fig = px.scatter(
             tsne_df,
@@ -106,7 +106,9 @@ def TSNEComponentsPairwisePlots(
                 "Component 1": "Component 1",
             },
         )
-        plots.append(fig)
+        figures.append(fig)
 
-    # Return the list of plots as a tuple
-    return tuple(plots)
+    return (
+        *figures,
+        RawData(embeddings_scaled=embeddings_scaled, tsne_results=tsne_results),
+    )

validmind/tests/model_validation/embeddings/utils.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import numpy as np
+import pandas as pd
 import plotly.express as px
 from sklearn.metrics.pairwise import cosine_similarity
 
@@ -17,10 +18,19 @@ def create_stability_analysis_result(
         original_embeddings, perturbed_embeddings
     ).diagonal()
 
+    # create a raw dataframe of the original, perturbed and similarity
+    raw_data = pd.DataFrame(
+        {
+            "original": original_embeddings,
+            "perturbed": perturbed_embeddings,
+            "similarity": similarities,
+        }
+    )
+
     mean = np.mean(similarities)
     passed = mean > mean_similarity_threshold
 
-    return (
+    return raw_data, (
         [
             {
                 "Mean Similarity": mean,

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -144,4 +144,5 @@ def AnswerCorrectness(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/ragas/AspectCritic.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -187,9 +187,13 @@ def AspectCritic(
         title="Aspect Critique Results",
     )
 
-    return {
-        "Aspect Scores": [
-            {"Aspect": aspect, "Score": result_df[aspect].mean()}
-            for aspect in aspects + [aspect.name for aspect in custom_aspects]
-        ]
-    }, fig
+    return (
+        {
+            "Aspect Scores": [
+                {"Aspect": aspect, "Score": result_df[aspect].mean()}
+                for aspect in aspects + [aspect.name for aspect in custom_aspects]
+            ]
+        },
+        fig,
+        RawData(evaluation_results=result_df),
+    )

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -143,4 +143,5 @@ def ContextEntityRecall(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -135,4 +135,5 @@ def ContextPrecision(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -130,4 +130,5 @@ def ContextPrecisionWithoutReference(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -135,4 +135,5 @@ def ContextRecall(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -140,4 +140,5 @@ def Faithfulness(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/ragas/NoiseSensitivity.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -179,4 +179,5 @@ def NoiseSensitivity(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/ragas/ResponseRelevancy.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -154,4 +154,5 @@ def ResponseRelevancy(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/ragas/SemanticSimilarity.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -133,4 +133,5 @@ def SemanticSimilarity(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/sklearn/CalibrationCurve.py

@@ -2,10 +2,11 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from sklearn.calibration import calibration_curve
 import plotly.graph_objects as go
+from sklearn.calibration import calibration_curve
+
 from validmind import tags, tasks
-from validmind.vm_models import VMModel, VMDataset
+from validmind.vm_models import VMDataset, VMModel
 from validmind.vm_models.result import RawData
 
 

validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py

@@ -6,11 +6,8 @@ import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
-from sklearn.metrics import (
-    roc_curve,
-    precision_recall_curve,
-    confusion_matrix,
-)
+from sklearn.metrics import confusion_matrix, precision_recall_curve, roc_curve
+
 from validmind import tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 

validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py

@@ -5,7 +5,7 @@
 import numpy as np
 from sklearn.metrics.pairwise import cosine_similarity
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -61,11 +61,14 @@ def ClusterCosineSimilarity(model: VMModel, dataset: VMDataset):
 
     table = []
 
+    cluster_centroids = {}
+
     for cluster_idx in range(num_clusters):
         cluster_data = dataset.x[y_pred == cluster_idx]
 
         if cluster_data.size != 0:
             cluster_centroid = np.mean(cluster_data, axis=0)
+            cluster_centroids[cluster_idx] = cluster_centroid
             table.append(
                 {
                     "Cluster": cluster_idx,
@@ -81,4 +84,4 @@ def ClusterCosineSimilarity(model: VMModel, dataset: VMDataset):
     if not table:
         raise SkipTestError("No clusters found")
 
-    return table
+    return table, RawData(cluster_centroids=cluster_centroids)