validmind 2.7.6__py3-none-any.whl → 2.7.8__py3-none-any.whl

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -140,4 +140,5 @@ def Faithfulness(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/ragas/NoiseSensitivity.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -179,4 +179,5 @@ def NoiseSensitivity(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/ragas/ResponseRelevancy.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -154,4 +154,5 @@ def ResponseRelevancy(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/ragas/SemanticSimilarity.py

@@ -7,7 +7,7 @@ import warnings
 import plotly.express as px
 from datasets import Dataset
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import MissingDependencyError
 
 from .utils import get_ragas_config, get_renamed_columns
@@ -133,4 +133,5 @@ def SemanticSimilarity(
         },
         fig_histogram,
         fig_box,
+        RawData(evaluation_results=result_df),
     )

validmind/tests/model_validation/sklearn/CalibrationCurve.py

@@ -2,10 +2,11 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from sklearn.calibration import calibration_curve
 import plotly.graph_objects as go
+from sklearn.calibration import calibration_curve
+
 from validmind import tags, tasks
-from validmind.vm_models import VMModel, VMDataset
+from validmind.vm_models import VMDataset, VMModel
 from validmind.vm_models.result import RawData
 
 

validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py

@@ -6,11 +6,8 @@ import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
-from sklearn.metrics import (
-    roc_curve,
-    precision_recall_curve,
-    confusion_matrix,
-)
+from sklearn.metrics import confusion_matrix, precision_recall_curve, roc_curve
+
 from validmind import tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 

validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py

@@ -5,7 +5,7 @@
 import numpy as np
 from sklearn.metrics.pairwise import cosine_similarity
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -61,11 +61,14 @@ def ClusterCosineSimilarity(model: VMModel, dataset: VMDataset):
 
     table = []
 
+    cluster_centroids = {}
+
     for cluster_idx in range(num_clusters):
         cluster_data = dataset.x[y_pred == cluster_idx]
 
         if cluster_data.size != 0:
             cluster_centroid = np.mean(cluster_data, axis=0)
+            cluster_centroids[cluster_idx] = cluster_centroid
             table.append(
                 {
                     "Cluster": cluster_idx,
@@ -81,4 +84,4 @@ def ClusterCosineSimilarity(model: VMModel, dataset: VMDataset):
     if not table:
         raise SkipTestError("No clusters found")
 
-    return table
+    return table, RawData(cluster_centroids=cluster_centroids)

validmind/tests/model_validation/sklearn/ConfusionMatrix.py

@@ -7,7 +7,7 @@ import numpy as np
 import plotly.figure_factory as ff
 from sklearn.metrics import confusion_matrix
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -119,4 +119,4 @@ def ConfusionMatrix(dataset: VMDataset, model: VMModel):
         font=dict(size=14),
     )
 
-    return fig
+    return fig, RawData(confusion_matrix=cm)

validmind/tests/model_validation/sklearn/FeatureImportance.py

@@ -52,8 +52,6 @@ def FeatureImportance(dataset: VMDataset, model: VMModel, num_features: int = 3)
     - The function's output is dependent on the number of features specified by `num_features`, which defaults to 3 but
     can be adjusted.
     """
-    results_list = []
-
     pfi_values = permutation_importance(
         estimator=model.model,
         X=dataset.x_df(),
@@ -61,8 +59,6 @@ def FeatureImportance(dataset: VMDataset, model: VMModel, num_features: int = 3)
         random_state=0,
         n_jobs=-2,
     )
-
-    # Create a dictionary to store PFI scores
     pfi = {
         column: pfi_values["importances_mean"][i]
         for i, column in enumerate(dataset.feature_columns)
@@ -70,14 +66,10 @@ def FeatureImportance(dataset: VMDataset, model: VMModel, num_features: int = 3)
 
     # Sort features by their importance
     sorted_features = sorted(pfi.items(), key=lambda item: item[1], reverse=True)
-
-    # Extract the top `num_features` features
     top_features = sorted_features[:num_features]
 
-    # Prepare the result for the current model and dataset
     result = {}
 
-    # Dynamically add feature columns to the result
     for i in range(num_features):
         if i < len(top_features):
             result[
@@ -86,9 +78,4 @@ def FeatureImportance(dataset: VMDataset, model: VMModel, num_features: int = 3)
         else:
             result[f"Feature {i + 1}"] = None
 
-    # Append the result to the list
-    results_list.append(result)
-
-    # Convert the results list to a DataFrame
-    results_df = pd.DataFrame(results_list)
-    return results_df
+    return pd.DataFrame([result])

validmind/tests/model_validation/sklearn/HyperParametersTuning.py

@@ -2,9 +2,10 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from typing import Union, Dict, List
-from sklearn.model_selection import GridSearchCV
+from typing import Dict, List, Union
+
 from sklearn.metrics import make_scorer, recall_score
+from sklearn.model_selection import GridSearchCV
 
 from validmind import tags, tasks
 from validmind.vm_models import VMDataset, VMModel
@@ -24,7 +25,9 @@ def _get_metrics(scoring):
     return (
         scoring
         if isinstance(scoring, list)
-        else list(scoring.keys()) if isinstance(scoring, dict) else [scoring]
+        else list(scoring.keys())
+        if isinstance(scoring, dict)
+        else [scoring]
     )
 
 

validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py

@@ -11,7 +11,7 @@ from scipy.spatial.distance import cdist
 from sklearn import clone
 from sklearn.metrics import silhouette_score
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -124,4 +124,4 @@ def KMeansClustersOptimization(
 
     fig.update_layout(showlegend=False)
 
-    return fig
+    return fig, RawData(distortions=distortions, silhouette_avg=silhouette_avg)

validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py

@@ -6,7 +6,7 @@ import numpy as np
 from sklearn.metrics import roc_auc_score
 from sklearn.preprocessing import LabelBinarizer
 
-from validmind.tests import tags, tasks
+from validmind import tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -62,14 +62,18 @@ def MinimumROCAUCScore(dataset: VMDataset, model: VMModel, min_threshold: float
         lb = LabelBinarizer()
         lb.fit(y_true)
 
+        y_true_binarized = lb.transform(y_true)
+        y_score_binarized = lb.transform(dataset.y_pred(model))
+
         roc_auc = roc_auc_score(
-            y_true=lb.transform(y_true),
-            y_score=lb.transform(dataset.y_pred(model)),
+            y_true=y_true_binarized,
+            y_score=y_score_binarized,
             average="macro",
         )
 
     else:
-        roc_auc = roc_auc_score(y_true=y_true, y_score=dataset.y_prob(model))
+        y_score_prob = dataset.y_prob(model)
+        roc_auc = roc_auc_score(y_true=y_true, y_score=y_score_prob)
 
     return [
         {

validmind/tests/model_validation/sklearn/ModelParameters.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import pandas as pd
+
 from validmind import tags, tasks
 
 

validmind/tests/model_validation/sklearn/OverfitDiagnosis.py

@@ -242,7 +242,7 @@ def OverfitDiagnosis(
         test_df[prob_column] = datasets[1].y_prob(model)
 
     test_results = []
-    test_figures = []
+    figures = []
     results_headers = ["slice", "shape", "feature", metric]
 
     for feature_column in datasets[0].feature_columns:
@@ -283,7 +283,7 @@ def OverfitDiagnosis(
             )
 
         results = _prepare_results(results_train, results_test, metric)
-        test_figures.append(
+        figures.append(
             _plot_overfit_regions(results, feature_column, cut_off_threshold, metric)
         )
 
@@ -299,4 +299,4 @@ def OverfitDiagnosis(
                 }
             )
 
-    return {"Overfit Diagnosis": test_results}, *test_figures
+    return ({"Overfit Diagnosis": test_results}, *figures)

validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py

@@ -7,7 +7,7 @@ from typing import Union
 import plotly.graph_objects as go
 from sklearn.inspection import permutation_importance
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
@@ -111,4 +111,4 @@ def PermutationFeatureImportance(
         height=figure_height,
     )
 
-    return fig
+    return fig, RawData(permutation_importance=pfi_values)

validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py

@@ -8,7 +8,7 @@ import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
@@ -192,18 +192,22 @@ def PopulationStabilityIndex(
 
     table_title = f"Population Stability Index for {datasets[0].input_id} and {datasets[1].input_id} Datasets"
 
-    return {
-        table_title: [
-            {
-                "Bin": (
-                    i if i < (len(psi_results) - 1) else "Total"
-                ),  # The last bin is the "Total" bin
-                "Count Initial": values["initial"],
-                "Percent Initial (%)": values["percent_initial"] * 100,
-                "Count New": values["new"],
-                "Percent New (%)": values["percent_new"] * 100,
-                "PSI": values["psi"],
-            }
-            for i, values in enumerate(psi_results)
-        ],
-    }, fig
+    return (
+        {
+            table_title: [
+                {
+                    "Bin": (
+                        i if i < (len(psi_results) - 1) else "Total"
+                    ),  # The last bin is the "Total" bin
+                    "Count Initial": values["initial"],
+                    "Percent Initial (%)": values["percent_initial"] * 100,
+                    "Count New": values["new"],
+                    "Percent New (%)": values["percent_new"] * 100,
+                    "PSI": values["psi"],
+                }
+                for i, values in enumerate(psi_results)
+            ],
+        },
+        fig,
+        RawData(psi_raw=psi_results),
+    )

validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py

@@ -6,7 +6,7 @@ import numpy as np
 import plotly.graph_objects as go
 from sklearn.metrics import precision_recall_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.models import FoundationModel
 from validmind.vm_models import VMDataset, VMModel
@@ -66,7 +66,7 @@ def PrecisionRecallCurve(model: VMModel, dataset: VMDataset):
 
     precision, recall, _ = precision_recall_curve(y_true, dataset.y_prob(model))
 
-    return go.Figure(
+    fig = go.Figure(
         data=[
             go.Scatter(
                 x=recall,
@@ -82,3 +82,5 @@ def PrecisionRecallCurve(model: VMModel, dataset: VMDataset):
             yaxis=dict(title="Precision"),
         ),
     )
+
+    return fig, RawData(precision=precision, recall=recall)

validmind/tests/model_validation/sklearn/ROCCurve.py

@@ -78,7 +78,6 @@ def ROCCurve(model: VMModel, dataset: VMDataset):
     auc = roc_auc_score(y_true, y_prob)
 
     return (
-        RawData(fpr=fpr, tpr=tpr, auc=auc),
         go.Figure(
             data=[
                 go.Scatter(
@@ -104,4 +103,5 @@ def ROCCurve(model: VMModel, dataset: VMDataset):
                 height=500,
             ),
         ),
+        RawData(fpr=fpr, tpr=tpr, auc=auc),
     )

validmind/tests/model_validation/sklearn/RegressionR2Square.py

@@ -51,17 +51,15 @@ def RegressionR2Square(dataset, model):
     violated.
     - Does not provide insight on whether the correct regression model was used or if key assumptions have been met.
     """
-
     y_true = dataset.y
     y_pred = dataset.y_pred(model)
     y_true = y_true.astype(y_pred.dtype)
 
-    r2s = metrics.r2_score(y_true, y_pred)
-    adj_r2 = adj_r2_score(y_true, y_pred, len(y_true), len(dataset.feature_columns))
-
-    # Create dataframe with R2 and Adjusted R2 in one row
-    results_df = pd.DataFrame(
-        {"R-squared (R2) Score": [r2s], "Adjusted R-squared (R2) Score": [adj_r2]}
+    return pd.DataFrame(
+        {
+            "R-squared (R2) Score": [metrics.r2_score(y_true, y_pred)],
+            "Adjusted R-squared (R2) Score": [
+                adj_r2_score(y_true, y_pred, len(y_true), len(dataset.feature_columns))
+            ],
+        }
     )
-
-    return results_df

validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.py

@@ -77,6 +77,4 @@ def RegressionR2SquareComparison(datasets, models):
             }
         )
 
-    # Convert results list to a DataFrame
-    results_df = pd.DataFrame(results_list)
-    return results_df
+    return pd.DataFrame(results_list)

validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py

@@ -9,7 +9,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 import shap
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import UnsupportedModelForSHAPError
 from validmind.logging import get_logger
 from validmind.models import CatBoostModel, SKlearnModel, StatsModelsModel
@@ -229,4 +229,5 @@ def SHAPGlobalImportance(
     return (
         generate_shap_plot("mean", shap_values, shap_sample),
         generate_shap_plot("summary", shap_values, shap_sample),
+        RawData(shap_values=shap_values, shap_sample=shap_sample),
     )

validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.py

@@ -4,8 +4,9 @@
 
 import pandas as pd
 import plotly.graph_objects as go
+
 from validmind import tags, tasks
-from validmind.vm_models import VMModel, VMDataset
+from validmind.vm_models import VMDataset, VMModel
 
 
 @tags("visualization", "credit_risk", "calibration")

validmind/tests/model_validation/sklearn/SilhouettePlot.py

@@ -6,7 +6,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 from sklearn.metrics import silhouette_samples, silhouette_score
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -105,8 +105,10 @@ def SilhouettePlot(model: VMModel, dataset: VMDataset):
 
     plt.close()
 
-    return [
+    return (
         {
             "Silhouette Score": silhouette_avg,
         },
-    ], fig
+        fig,
+        RawData(sample_silhouette_values=sample_silhouette_values),
+    )

validmind/tests/model_validation/sklearn/TrainingTestDegradation.py

@@ -7,6 +7,7 @@ from typing import List
 from numpy import unique
 from sklearn.metrics import classification_report
 
+from validmind import RawData
 from validmind.tests import tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
@@ -98,4 +99,11 @@ def TrainingTestDegradation(
                 }
             )
 
-    return table, all(row["Pass/Fail"] == "Pass" for row in table)
+    return (
+        table,
+        all(row["Pass/Fail"] == "Pass" for row in table),
+        RawData(
+            dataset_1_report=ds1_report,
+            dataset_2_report=ds2_report,
+        ),
+    )

validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py

@@ -9,7 +9,7 @@ import pandas as pd
 import seaborn as sns
 from sklearn import metrics
 
-from validmind.tests import tags, tasks
+from validmind import tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 DEFAULT_METRICS = {

validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py

@@ -6,7 +6,7 @@ import numpy as np
 import plotly.graph_objects as go
 from matplotlib import cm
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization", "credit_risk")
@@ -62,9 +62,9 @@ def CumulativePredictionProbabilities(dataset, model, title="Cumulative Probabil
     df = dataset.df
     df["probabilities"] = dataset.y_prob(model)
 
-    fig = _plot_cumulative_prob(df, dataset.target_column, title)
+    fig, fig_data = _plot_cumulative_prob(df, dataset.target_column, title)
 
-    return fig
+    return fig, RawData(cumulative_probabilities=fig_data)
 
 
 def _plot_cumulative_prob(df, target_col, title):
@@ -82,10 +82,17 @@ def _plot_cumulative_prob(df, target_col, title):
         cls: f"rgb({int(rgb[0]*255)}, {int(rgb[1]*255)}, {int(rgb[2]*255)})"
         for cls, rgb in zip(classes, colors)
     }
+
+    raw_data = {}
+
     for class_value in sorted(df[target_col].unique()):
         # Calculate cumulative distribution for the current class
         sorted_probs = np.sort(df[df[target_col] == class_value]["probabilities"])
         cumulative_probs = np.cumsum(sorted_probs) / np.sum(sorted_probs)
+        raw_data[class_value] = {
+            "sorted_probs": sorted_probs,
+            "cumulative_probs": cumulative_probs,
+        }
 
         fig.add_trace(
             go.Scatter(
@@ -104,4 +111,4 @@ def _plot_cumulative_prob(df, target_col, title):
             yaxis_title="Cumulative Distribution",
         )
 
-    return fig
+    return fig, raw_data

validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py

@@ -75,12 +75,10 @@ def DurbinWatsonTest(dataset, model, threshold=[1.5, 2.5]):
         else:
             return "No autocorrelation"
 
-    results = pd.DataFrame(
+    return pd.DataFrame(
         {
             "dw_statistic": [dw_statistic],
             "threshold": [str(threshold)],
             "autocorrelation": [get_autocorrelation(dw_statistic, threshold)],
         }
     )
-
-    return results

validmind/tests/model_validation/statsmodels/GINITable.py

@@ -61,27 +61,19 @@ def GINITable(dataset, model):
     - The test does not incorporate a method to efficiently handle missing or inefficiently processed data, which could
     lead to inaccuracies in the metrics if the data is not appropriately preprocessed.
     """
-
-    metrics_dict = {"AUC": [], "GINI": [], "KS": []}
-
-    # Retrieve y_true and y_pred for the current dataset
     y_true = np.ravel(dataset.y)  # Flatten y_true to make it one-dimensional
     y_prob = dataset.y_prob(model)
-
-    # Compute metrics
     y_true = np.array(y_true, dtype=float)
     y_prob = np.array(y_prob, dtype=float)
 
     fpr, tpr, _ = roc_curve(y_true, y_prob)
-    ks = max(tpr - fpr)
     auc = roc_auc_score(y_true, y_prob)
     gini = 2 * auc - 1
 
-    # Add the metrics to the dictionary
-    metrics_dict["AUC"].append(auc)
-    metrics_dict["GINI"].append(gini)
-    metrics_dict["KS"].append(ks)
-
-    # Create a DataFrame to store and return the results
-    metrics_df = pd.DataFrame(metrics_dict)
-    return metrics_df
+    return pd.DataFrame(
+        {
+            "AUC": [auc],
+            "GINI": [gini],
+            "KS": [max(tpr - fpr)],
+        }
+    )

validmind/tests/model_validation/statsmodels/Lilliefors.py

@@ -5,12 +5,12 @@
 from statsmodels.stats.diagnostic import lilliefors
 
 from validmind import tags, tasks
-from validmind.vm_models import VMDataset, VMModel
+from validmind.vm_models import VMDataset
 
 
 @tags("tabular_data", "data_distribution", "statistical_test", "statsmodels")
 @tasks("classification", "regression")
-def Lilliefors(model: VMModel, dataset: VMDataset):
+def Lilliefors(dataset: VMDataset):
     """
     Assesses the normality of feature distributions in an ML model's training dataset using the Lilliefors test.
 

validmind/tests/model_validation/statsmodels/RegressionCoeffs.py

@@ -97,4 +97,4 @@ def RegressionCoeffs(model):
         yaxis_title="Coefficients",
     )
 
-    return (fig, coefficients)
+    return fig, coefficients

validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py

@@ -5,7 +5,7 @@
 import matplotlib.pyplot as plt
 import seaborn as sns
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.logging import get_logger
 from validmind.vm_models import VMModel
@@ -90,4 +90,4 @@ def RegressionFeatureSignificance(
 
     plt.close()
 
-    return fig
+    return fig, RawData(coefficients=coefficients, pvalues=pvalues)

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py

@@ -6,7 +6,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -91,4 +91,7 @@ def RegressionModelForecastPlotLevels(
 
     plt.close()
 
-    return fig
+    return fig, RawData(
+        y_transformed=dataset_y_transformed,
+        y_pred_transformed=y_pred_transformed,
+    )