validmind 2.8.10__py3-none-any.whl → 2.8.12__py3-none-any.whl

validmind/tests/model_validation/FeaturesAUC.py

@@ -95,4 +95,4 @@ def FeaturesAUC(dataset: VMDataset, fontsize: int = 12, figure_height: int = 500
         height=figure_height,
     )
 
-    return fig, RawData(feature_aucs=aucs)
+    return fig, RawData(feature_aucs=aucs, dataset=dataset.input_id)

validmind/tests/model_validation/MeteorScore.py

@@ -117,4 +117,10 @@ def MeteorScore(dataset, model):
     # Create a DataFrame from all collected statistics
     result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
 
-    return (result_df, *figures, RawData(meteor_scores=metrics_df))
+    return (
+        result_df,
+        *figures,
+        RawData(
+            meteor_scores=metrics_df, model=model.input_id, dataset=dataset.input_id
+        ),
+    )

validmind/tests/model_validation/ModelPredictionResiduals.py

@@ -102,4 +102,8 @@ def ModelPredictionResiduals(
     # Create a summary DataFrame for the KS normality test results
     summary_df = pd.DataFrame([summary])
 
-    return (summary_df, *figures, RawData(residuals=residuals))
+    return (
+        summary_df,
+        *figures,
+        RawData(residuals=residuals, model=model.input_id, dataset=dataset.input_id),
+    )

validmind/tests/model_validation/RegardScore.py

@@ -145,5 +145,10 @@ def RegardScore(dataset, model):
     return (
         result_df,
         *figures,
-        RawData(true_regard=true_df, pred_regard=pred_df),
+        RawData(
+            true_regard=true_df,
+            pred_regard=pred_df,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/model_validation/RegressionResidualsPlot.py

@@ -105,4 +105,13 @@ def RegressionResidualsPlot(model: VMModel, dataset: VMDataset, bin_size: float
         )
     )
 
-    return (*figures, RawData(residuals=residuals, y_true=y_true, y_pred=y_pred))
+    return (
+        *figures,
+        RawData(
+            residuals=residuals,
+            y_true=y_true,
+            y_pred=y_pred,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
+    )

validmind/tests/model_validation/RougeScore.py

@@ -121,5 +121,7 @@ def RougeScore(dataset, model, metric="rouge-1"):
     return (
         pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"}),
         *figures,
-        RawData(rouge_scores_df=df_scores),
+        RawData(
+            rouge_scores_df=df_scores, model=model.input_id, dataset=dataset.input_id
+        ),
     )

validmind/tests/model_validation/TimeSeriesPredictionWithCI.py

@@ -152,5 +152,7 @@ def TimeSeriesPredictionWithCI(dataset, model, confidence=0.95):
             z_score=z_score,
             lower_confidence=lower_conf,
             upper_confidence=upper_conf,
+            model=model.input_id,
+            dataset=dataset.input_id,
         ),
     )

validmind/tests/model_validation/TimeSeriesPredictionsPlot.py

@@ -4,7 +4,7 @@
 
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("model_predictions", "visualization")
@@ -70,4 +70,12 @@ def TimeSeriesPredictionsPlot(dataset, model):
         template="plotly_white",
     )
 
-    return fig
+    raw_data = RawData(
+        time_index=time_index,
+        actual_values=dataset.y,
+        predicted_values=y_pred,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
+
+    return fig, raw_data

validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py

@@ -7,7 +7,7 @@ import pandas as pd
 import plotly.express as px
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("model_performance", "sklearn")
@@ -105,4 +105,8 @@ def TimeSeriesR2SquareBySegments(dataset, model, segments=None):
         },
     )
 
-    return fig, results_df
+    return (
+        fig,
+        results_df,
+        RawData(summary=results_df, model=model.input_id, dataset=dataset.input_id),
+    )

validmind/tests/model_validation/TokenDisparity.py

@@ -108,4 +108,8 @@ def TokenDisparity(dataset, model):
     # Create a DataFrame from all collected statistics
     result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
 
-    return (result_df, *figures, RawData(token_counts_df=df))
+    return (
+        result_df,
+        *figures,
+        RawData(token_counts_df=df, model=model.input_id, dataset=dataset.input_id),
+    )

validmind/tests/model_validation/ToxicityScore.py

@@ -146,5 +146,7 @@ def ToxicityScore(dataset, model):
             input_toxicity_df=input_df,
             true_toxicity_df=true_df,
             pred_toxicity_df=pred_df,
+            model=model.input_id,
+            dataset=dataset.input_id,
         ),
     )

validmind/tests/model_validation/embeddings/ClusterDistribution.py

@@ -62,4 +62,4 @@ def ClusterDistribution(model: VMModel, dataset: VMDataset, num_clusters: int =
         title="Embeddings Cluster Distribution",
     )
 
-    return fig, RawData(labels=labels)
+    return fig, RawData(labels=labels, model=model.input_id, dataset=dataset.input_id)

validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py

@@ -113,5 +113,9 @@ def CosineSimilarityComparison(dataset, models):
     return (
         *figures,
         stats_df,
-        RawData(similarity_matrices=pd.DataFrame(similarity_matrices)),
+        RawData(
+            similarity_matrices=pd.DataFrame(similarity_matrices),
+            dataset=dataset.input_id,
+            models=[model.input_id for model in models],
+        ),
     )

validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py

@@ -59,4 +59,8 @@ def CosineSimilarityDistribution(dataset: VMDataset, model: VMModel):
         nbins=100,
         title="Cosine Similarity Distribution",
         labels={"x": "Cosine Similarity"},
-    ), RawData(similarity_scores=similarity_scores)
+    ), RawData(
+        similarity_scores=similarity_scores,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py

@@ -81,4 +81,8 @@ def CosineSimilarityHeatmap(
         yaxis_title=yaxis_title,
     )
 
-    return fig, RawData(similarity_matrix=similarity_matrix)
+    return fig, RawData(
+        similarity_matrix=similarity_matrix,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py

@@ -77,5 +77,7 @@ def DescriptiveAnalytics(dataset: VMDataset, model: VMModel):
             embedding_means=embedding_means,
             embedding_medians=embedding_medians,
             embedding_stds=embedding_stds,
+            model=model.input_id,
+            dataset=dataset.input_id,
         ),
     )

validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py

@@ -89,4 +89,8 @@ def EmbeddingsVisualization2D(
     fig = px.scatter(**scatter_kwargs)
     fig.update_layout(width=500, height=500)
 
-    return fig, RawData(tsne_embeddings=reduced_embeddings)
+    return fig, RawData(
+        tsne_embeddings=reduced_embeddings,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py

@@ -57,7 +57,7 @@ def EuclideanDistanceComparison(dataset, models):
     figures = []
     all_stats = []
 
-    distance_matrices = {}
+    distance_matrices = []
 
     # Generate all pairs of models for comparison
     for model_A, model_B in combinations(models, 2):
@@ -105,6 +105,10 @@ def EuclideanDistanceComparison(dataset, models):
     stats_df = pd.DataFrame(all_stats)
 
     # Add raw data to return
-    raw_data = RawData(distance_matrices=pd.DataFrame(distance_matrices))
+    raw_data = RawData(
+        distance_matrices=pd.DataFrame(distance_matrices),
+        dataset=dataset.input_id,
+        models=[model.input_id for model in models],
+    )
 
     return (stats_df, *figures, raw_data)

validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py

@@ -79,4 +79,6 @@ def EuclideanDistanceHeatmap(
         yaxis_title=yaxis_title,
     )
 
-    return fig, RawData(distance_matrix=distance_matrix)
+    return fig, RawData(
+        distance_matrix=distance_matrix, model=model.input_id, dataset=dataset.input_id
+    )

validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py

@@ -90,4 +90,7 @@ def PCAComponentsPairwisePlots(dataset, model, n_components=3):
         )
         figures.append(fig)
 
-    return (*figures, RawData(pca_results=pca_df))
+    return (
+        *figures,
+        RawData(pca_results=pca_df, model=model.input_id, dataset=dataset.input_id),
+    )

validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py

@@ -97,4 +97,8 @@ def StabilityAnalysisKeyword(
         mean_similarity_threshold,
     )
 
-    return results, RawData(original_perturbed_similarity=raw_data)
+    return results, RawData(
+        original_perturbed_similarity=raw_data,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py

@@ -151,4 +151,8 @@ def StabilityAnalysisRandomNoise(
         mean_similarity_threshold,
     )
 
-    return *result, RawData(original_perturbed_similarity=raw_data)
+    return *result, RawData(
+        original_perturbed_similarity=raw_data,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py

@@ -107,4 +107,8 @@ def StabilityAnalysisSynonyms(
         mean_similarity_threshold,
     )
 
-    return *result, RawData(original_perturbed_similarity=raw_data)
+    return *result, RawData(
+        original_perturbed_similarity=raw_data,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py

@@ -134,4 +134,8 @@ def StabilityAnalysisTranslation(
         mean_similarity_threshold,
     )
 
-    return *result, RawData(original_perturbed_similarity=raw_data)
+    return *result, RawData(
+        original_perturbed_similarity=raw_data,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py

@@ -110,5 +110,10 @@ def TSNEComponentsPairwisePlots(
 
     return (
         *figures,
-        RawData(embeddings_scaled=embeddings_scaled, tsne_results=tsne_results),
+        RawData(
+            embeddings_scaled=embeddings_scaled,
+            tsne_results=tsne_results,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -144,5 +144,5 @@ def AnswerCorrectness(
         },
         fig_histogram,
         fig_box,
-        RawData(evaluation_results=result_df),
+        RawData(evaluation_results=result_df, dataset=dataset.input_id),
     )

validmind/tests/model_validation/ragas/AspectCritic.py

@@ -195,5 +195,8 @@ def AspectCritic(
             ]
         },
         fig,
-        RawData(evaluation_results=result_df),
+        RawData(
+            evaluation_results=result_df,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -143,5 +143,5 @@ def ContextEntityRecall(
         },
         fig_histogram,
         fig_box,
-        RawData(evaluation_results=result_df),
+        RawData(evaluation_results=result_df, dataset=dataset.input_id),
     )

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -135,5 +135,5 @@ def ContextPrecision(
         },
         fig_histogram,
         fig_box,
-        RawData(evaluation_results=result_df),
+        RawData(evaluation_results=result_df, dataset=dataset.input_id),
     )

validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py

@@ -130,5 +130,5 @@ def ContextPrecisionWithoutReference(
         },
         fig_histogram,
         fig_box,
-        RawData(evaluation_results=result_df),
+        RawData(evaluation_results=result_df, dataset=dataset.input_id),
     )

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -135,5 +135,5 @@ def ContextRecall(
         },
         fig_histogram,
         fig_box,
-        RawData(evaluation_results=result_df),
+        RawData(evaluation_results=result_df, dataset=dataset.input_id),
     )

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -140,5 +140,5 @@ def Faithfulness(
         },
         fig_histogram,
         fig_box,
-        RawData(evaluation_results=result_df),
+        RawData(evaluation_results=result_df, dataset=dataset.input_id),
     )

validmind/tests/model_validation/ragas/NoiseSensitivity.py

@@ -179,5 +179,5 @@ def NoiseSensitivity(
         },
         fig_histogram,
         fig_box,
-        RawData(evaluation_results=result_df),
+        RawData(evaluation_results=result_df, dataset=dataset.input_id),
     )

validmind/tests/model_validation/ragas/ResponseRelevancy.py

@@ -154,5 +154,5 @@ def ResponseRelevancy(
         },
         fig_histogram,
         fig_box,
-        RawData(evaluation_results=result_df),
+        RawData(evaluation_results=result_df, dataset=dataset.input_id),
     )

validmind/tests/model_validation/ragas/SemanticSimilarity.py

@@ -133,5 +133,5 @@ def SemanticSimilarity(
         },
         fig_histogram,
         fig_box,
-        RawData(evaluation_results=result_df),
+        RawData(evaluation_results=result_df, dataset=dataset.input_id),
     )

validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py

@@ -4,7 +4,7 @@
 
 from sklearn.metrics import adjusted_mutual_info_score
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -52,11 +52,11 @@ def AdjustedMutualInformation(model: VMModel, dataset: VMDataset):
     - The interpretability of the score can be complex as it depends on the understanding of information theory
     concepts.
     """
-    return [
-        {
-            "Adjusted Mutual Information": adjusted_mutual_info_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
-            )
-        }
-    ]
+    ami_score = adjusted_mutual_info_score(
+        labels_true=dataset.y,
+        labels_pred=dataset.y_pred(model),
+    )
+
+    return [{"Adjusted Mutual Information": ami_score}], RawData(
+        ami_score=ami_score, model=model.input_id, dataset=dataset.input_id
+    )

validmind/tests/model_validation/sklearn/AdjustedRandIndex.py

@@ -4,7 +4,7 @@
 
 from sklearn.metrics import adjusted_rand_score
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -49,11 +49,11 @@ def AdjustedRandIndex(model: VMModel, dataset: VMDataset):
     - It may be difficult to interpret the implications of an ARI score without context or a benchmark, as it is
     heavily dependent on the characteristics of the dataset used.
     """
-    return [
-        {
-            "Adjusted Rand Index": adjusted_rand_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
-            )
-        }
-    ]
+    ari = adjusted_rand_score(
+        labels_true=dataset.y,
+        labels_pred=dataset.y_pred(model),
+    )
+
+    return [{"Adjusted Rand Index": ari}], RawData(
+        ari_score=ari, model=model.input_id, dataset=dataset.input_id
+    )

validmind/tests/model_validation/sklearn/CalibrationCurve.py

@@ -72,7 +72,10 @@ def CalibrationCurve(model: VMModel, dataset: VMDataset, n_bins: int = 10):
 
     # Create DataFrame for raw data
     raw_data = RawData(
-        mean_predicted_probability=prob_pred, observed_frequency=prob_true
+        mean_predicted_probability=prob_pred,
+        observed_frequency=prob_true,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )
 
     # Create Plotly figure
@@ -114,4 +117,4 @@ def CalibrationCurve(model: VMModel, dataset: VMDataset, n_bins: int = 10):
         template="plotly_white",
     )
 
-    return raw_data, fig
+    return fig, raw_data

validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py

@@ -8,7 +8,7 @@ import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 from sklearn.metrics import confusion_matrix, precision_recall_curve, roc_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -255,4 +255,17 @@ def ClassifierThresholdOptimization(
     # Create results table and sort by threshold descending
     table = pd.DataFrame(results).sort_values("threshold", ascending=False)
 
-    return fig, table
+    return (
+        fig,
+        table,
+        RawData(
+            fpr=fpr,
+            tpr=tpr,
+            precision=precision,
+            recall=recall,
+            thresholds_roc=thresholds_roc,
+            thresholds_pr=thresholds_pr,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
+    )

validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py

@@ -84,4 +84,8 @@ def ClusterCosineSimilarity(model: VMModel, dataset: VMDataset):
     if not table:
         raise SkipTestError("No clusters found")
 
-    return table, RawData(cluster_centroids=cluster_centroids)
+    return table, RawData(
+        cluster_centroids=cluster_centroids,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py

@@ -11,7 +11,7 @@ from sklearn.metrics import (
     v_measure_score,
 )
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 HOMOGENEITY = """
@@ -115,53 +115,63 @@ def ClusterPerformanceMetrics(model: VMModel, dataset: VMDataset):
     - Does not consider aspects like computational efficiency of the model or its capability to handle high dimensional
     data.
     """
-    return [
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    metrics = [
         {
             "Metric": "Homogeneity Score",
             "Description": HOMOGENEITY,
             "Value": homogeneity_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
+                labels_true=y_true,
+                labels_pred=y_pred,
             ),
         },
         {
             "Metric": "Completeness Score",
             "Description": COMPLETENESS,
             "Value": completeness_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
+                labels_true=y_true,
+                labels_pred=y_pred,
             ),
         },
         {
             "Metric": "V Measure",
             "Description": V_MEASURE,
             "Value": v_measure_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
+                labels_true=y_true,
+                labels_pred=y_pred,
             ),
         },
         {
             "Metric": "Adjusted Rand Index",
             "Description": ADJUSTED_RAND_INDEX,
             "Value": adjusted_rand_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
+                labels_true=y_true,
+                labels_pred=y_pred,
             ),
         },
         {
             "Metric": "Adjusted Mutual Information",
             "Description": ADJUSTED_MUTUAL_INFORMATION,
             "Value": adjusted_mutual_info_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
+                labels_true=y_true,
+                labels_pred=y_pred,
             ),
         },
         {
             "Metric": "Fowlkes-Mallows score",
             "Description": FOULKES_MALLOWS_SCORE,
             "Value": fowlkes_mallows_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
+                labels_true=y_true,
+                labels_pred=y_pred,
             ),
         },
     ]
+
+    return metrics, RawData(
+        true_labels=y_true,
+        predicted_labels=y_pred,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/sklearn/CompletenessScore.py

@@ -4,7 +4,7 @@
 
 from sklearn.metrics import completeness_score
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -47,11 +47,10 @@ def CompletenessScore(model: VMModel, dataset: VMDataset):
     - The Completeness Score only applies to clustering models; it cannot be used for other types of machine learning
     models.
     """
-    return [
-        {
-            "Completeness Score": completeness_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
-            )
-        }
-    ]
+    score = completeness_score(
+        labels_true=dataset.y,
+        labels_pred=dataset.y_pred(model),
+    )
+    return [{"Completeness Score": score}], RawData(
+        score=score, model=model.input_id, dataset=dataset.input_id
+    )