validmind 2.8.10__py3-none-any.whl → 2.8.12__py3-none-any.whl

validmind/tests/model_validation/sklearn/ConfusionMatrix.py

@@ -19,7 +19,11 @@ from validmind.vm_models import VMDataset, VMModel
     "visualization",
 )
 @tasks("classification", "text_classification")
-def ConfusionMatrix(dataset: VMDataset, model: VMModel):
+def ConfusionMatrix(
+    dataset: VMDataset,
+    model: VMModel,
+    threshold: float = 0.5,
+):
     """
     Evaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix
     heatmap.
@@ -66,7 +70,17 @@ def ConfusionMatrix(dataset: VMDataset, model: VMModel):
     - Risks of misinterpretation exist because the matrix doesn't directly provide precision, recall, or F1-score data.
     These metrics have to be computed separately.
     """
-    y_pred = dataset.y_pred(model)
+    # Get predictions using threshold for binary classification if possible
+    if hasattr(model.model, "predict_proba"):
+        y_prob = dataset.y_prob(model)
+        # Handle both 1D and 2D probability arrays
+        if y_prob.ndim == 2:
+            y_pred = (y_prob[:, 1] > threshold).astype(int)
+        else:
+            y_pred = (y_prob > threshold).astype(int)
+    else:
+        y_pred = dataset.y_pred(model)
+
     y_true = dataset.y.astype(y_pred.dtype)
 
     labels = np.unique(y_true)
@@ -119,4 +133,9 @@ def ConfusionMatrix(dataset: VMDataset, model: VMModel):
         font=dict(size=14),
     )
 
-    return fig, RawData(confusion_matrix=cm)
+    return fig, RawData(
+        confusion_matrix=cm,
+        threshold=threshold,
+        dataset=dataset.input_id,
+        model=model.input_id,
+    )

validmind/tests/model_validation/sklearn/FeatureImportance.py

@@ -5,7 +5,7 @@
 import pandas as pd
 from sklearn.inspection import permutation_importance
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -78,4 +78,8 @@ def FeatureImportance(dataset: VMDataset, model: VMModel, num_features: int = 3)
         else:
             result[f"Feature {i + 1}"] = None
 
-    return pd.DataFrame([result])
+    return pd.DataFrame([result]), RawData(
+        permutation_importance=pfi_values,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/sklearn/FowlkesMallowsScore.py

@@ -4,7 +4,7 @@
 
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -52,11 +52,14 @@ def FowlkesMallowsScore(dataset: VMDataset, model: VMModel):
     - It does not handle mismatching numbers of clusters between the true and predicted labels. As such, it may return
     misleading results if the predicted labels suggest a different number of clusters than what is in the true labels.
     """
-    return [
-        {
-            "Fowlkes-Mallows score": metrics.fowlkes_mallows_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
-            )
-        }
-    ]
+    fowlkes_mallows_score = metrics.fowlkes_mallows_score(
+        labels_true=dataset.y,
+        labels_pred=dataset.y_pred(model),
+    )
+
+    return [{"Fowlkes-Mallows score": fowlkes_mallows_score}], RawData(
+        labels_true=dataset.y,
+        labels_pred=dataset.y_pred(model),
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/sklearn/HomogeneityScore.py

@@ -4,7 +4,7 @@
 
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -50,11 +50,16 @@ def HomogeneityScore(dataset: VMDataset, model: VMModel):
     - The score does not address the actual number of clusters formed, or the evenness of cluster sizes. It only checks
     the homogeneity within the given clusters created by the model.
     """
-    return [
-        {
-            "Homogeneity Score": metrics.homogeneity_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
-            )
-        }
-    ]
+    homogeneity_score = metrics.homogeneity_score(
+        labels_true=dataset.y,
+        labels_pred=dataset.y_pred(model),
+    )
+
+    raw_data = RawData(
+        y_true=dataset.y,
+        y_pred=dataset.y_pred(model),
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
+
+    return ([{"Homogeneity Score": homogeneity_score}], raw_data)

validmind/tests/model_validation/sklearn/HyperParametersTuning.py

@@ -7,7 +7,7 @@ from typing import Dict, List, Union
 from sklearn.metrics import make_scorer, recall_score
 from sklearn.model_selection import GridSearchCV
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -162,4 +162,6 @@ def HyperParametersTuning(
 
             results.append(row_result)
 
-    return results
+    return results, RawData(
+        model=model.input_id, dataset=dataset.input_id, param_grid=param_grid
+    )

validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py

@@ -124,4 +124,9 @@ def KMeansClustersOptimization(
 
     fig.update_layout(showlegend=False)
 
-    return fig, RawData(distortions=distortions, silhouette_avg=silhouette_avg)
+    return fig, RawData(
+        distortions=distortions,
+        silhouette_avg=silhouette_avg,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/sklearn/MinimumAccuracy.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 from sklearn.metrics import accuracy_score
 
+from validmind import RawData
 from validmind.tests import tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
@@ -50,10 +51,14 @@ def MinimumAccuracy(dataset: VMDataset, model: VMModel, min_threshold: float = 0
     """
     accuracy = accuracy_score(dataset.y, dataset.y_pred(model))
 
-    return [
-        {
-            "Score": accuracy,
-            "Threshold": min_threshold,
-            "Pass/Fail": "Pass" if accuracy > min_threshold else "Fail",
-        }
-    ], accuracy > min_threshold
+    return (
+        [
+            {
+                "Score": accuracy,
+                "Threshold": min_threshold,
+                "Pass/Fail": "Pass" if accuracy > min_threshold else "Fail",
+            }
+        ],
+        accuracy > min_threshold,
+        RawData(model=model.input_id, dataset=dataset.input_id),
+    )

validmind/tests/model_validation/sklearn/MinimumF1Score.py

@@ -5,6 +5,7 @@
 import numpy as np
 from sklearn.metrics import f1_score
 
+from validmind import RawData
 from validmind.tests import tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
@@ -58,10 +59,14 @@ def MinimumF1Score(dataset: VMDataset, model: VMModel, min_threshold: float = 0.
     else:
         score = f1_score(dataset.y, dataset.y_pred(model))
 
-    return [
-        {
-            "Score": score,
-            "Threshold": min_threshold,
-            "Pass/Fail": "Pass" if score > min_threshold else "Fail",
-        }
-    ], score > min_threshold
+    return (
+        [
+            {
+                "Score": score,
+                "Threshold": min_threshold,
+                "Pass/Fail": "Pass" if score > min_threshold else "Fail",
+            }
+        ],
+        score > min_threshold,
+        RawData(score=score, model=model.input_id, dataset=dataset.input_id),
+    )

validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py

@@ -6,7 +6,7 @@ import numpy as np
 from sklearn.metrics import roc_auc_score
 from sklearn.preprocessing import LabelBinarizer
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -62,19 +62,34 @@ def MinimumROCAUCScore(dataset: VMDataset, model: VMModel, min_threshold: float
         lb = LabelBinarizer()
         lb.fit(y_true)
 
+        y_true_binary = lb.transform(y_true)
+        y_score_binary = lb.transform(dataset.y_pred(model))
+
         roc_auc = roc_auc_score(
-            y_true=lb.transform(y_true),
-            y_score=lb.transform(dataset.y_pred(model)),
+            y_true=y_true_binary,
+            y_score=y_score_binary,
             average="macro",
         )
 
     else:
-        roc_auc = roc_auc_score(y_true=y_true, y_score=dataset.y_prob(model))
+        y_score_prob = dataset.y_prob(model)
+        roc_auc = roc_auc_score(y_true=y_true, y_score=y_score_prob)
 
-    return [
+    results = [
         {
             "Score": roc_auc,
             "Threshold": min_threshold,
             "Pass/Fail": "Pass" if roc_auc > min_threshold else "Fail",
         }
-    ], roc_auc > min_threshold
+    ]
+
+    return (
+        results,
+        roc_auc > min_threshold,
+        RawData(
+            y_true=y_true,
+            roc_auc=roc_auc,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
+    )

validmind/tests/model_validation/sklearn/OverfitDiagnosis.py

@@ -10,7 +10,7 @@ import pandas as pd
 import seaborn as sns
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
@@ -299,4 +299,10 @@ def OverfitDiagnosis(
                 }
             )
 
-    return ({"Overfit Diagnosis": test_results}, *figures)
+    return (
+        {"Overfit Diagnosis": test_results},
+        *figures,
+        RawData(
+            model=model.input_id, datasets=[dataset.input_id for dataset in datasets]
+        ),
+    )

validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py

@@ -111,4 +111,8 @@ def PermutationFeatureImportance(
         height=figure_height,
     )
 
-    return fig, RawData(permutation_importance=pfi_values)
+    return fig, RawData(
+        permutation_importance=pfi_values,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py

@@ -209,5 +209,9 @@ def PopulationStabilityIndex(
             ],
         },
         fig,
-        RawData(psi_raw=psi_results),
+        RawData(
+            psi_raw=psi_results,
+            model=model.input_id,
+            datasets=[datasets[0].input_id, datasets[1].input_id],
+        ),
     )

validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py

@@ -83,4 +83,9 @@ def PrecisionRecallCurve(model: VMModel, dataset: VMDataset):
         ),
     )
 
-    return fig, RawData(precision=precision, recall=recall)
+    return fig, RawData(
+        precision=precision,
+        recall=recall,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/sklearn/ROCCurve.py

@@ -103,5 +103,7 @@ def ROCCurve(model: VMModel, dataset: VMDataset):
                 height=500,
             ),
         ),
-        RawData(fpr=fpr, tpr=tpr, auc=auc),
+        RawData(
+            fpr=fpr, tpr=tpr, auc=auc, model=model.input_id, dataset=dataset.input_id
+        ),
     )

validmind/tests/model_validation/sklearn/RegressionErrors.py

@@ -6,7 +6,7 @@ import numpy as np
 import pandas as pd
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("sklearn", "model_performance")
@@ -62,7 +62,11 @@ def RegressionErrors(model, dataset):
     y_pred = dataset.y_pred(model)
     y_true = y_true.astype(y_pred.dtype)
 
-    return _regression_errors(y_true, y_pred)
+    results_df = _regression_errors(y_true, y_pred)
+
+    return results_df, RawData(
+        y_true=y_true, y_pred=y_pred, model=model.input_id, dataset=dataset.input_id
+    )
 
 
 def _regression_errors(y_true, y_pred):

validmind/tests/model_validation/sklearn/RegressionPerformance.py

@@ -5,7 +5,7 @@
 import numpy as np
 from sklearn.metrics import mean_absolute_error, mean_squared_error
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
@@ -74,10 +74,15 @@ def RegressionPerformance(model: VMModel, dataset: VMDataset):
     # MBD calculation
     metrics["Mean Bias Deviation (MBD)"] = np.mean(y_pred - y_true)
 
-    return [
-        {
-            "Metric": metric,
-            "Value": value,
-        }
-        for metric, value in metrics.items()
-    ]
+    return (
+        [
+            {
+                "Metric": metric,
+                "Value": value,
+            }
+            for metric, value in metrics.items()
+        ],
+        RawData(
+            y_true=y_true, y_pred=y_pred, model=model.input_id, dataset=dataset.input_id
+        ),
+    )

validmind/tests/model_validation/sklearn/RegressionR2Square.py

@@ -5,7 +5,7 @@
 import pandas as pd
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.tests.model_validation.statsmodels.statsutils import adj_r2_score
 
 
@@ -55,11 +55,14 @@ def RegressionR2Square(dataset, model):
     y_pred = dataset.y_pred(model)
     y_true = y_true.astype(y_pred.dtype)
 
+    r2 = metrics.r2_score(y_true, y_pred)
+    adj_r2 = adj_r2_score(y_true, y_pred, len(y_true), len(dataset.feature_columns))
+
     return pd.DataFrame(
         {
-            "R-squared (R2) Score": [metrics.r2_score(y_true, y_pred)],
-            "Adjusted R-squared (R2) Score": [
-                adj_r2_score(y_true, y_pred, len(y_true), len(dataset.feature_columns))
-            ],
+            "R-squared (R2) Score": [r2],
+            "Adjusted R-squared (R2) Score": [adj_r2],
         }
+    ), RawData(
+        y_true=y_true, y_pred=y_pred, model=model.input_id, dataset=dataset.input_id
     )

validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py

@@ -327,4 +327,8 @@ def RobustnessDiagnosis(
         results_df["Perturbation Size"] == 0.0, "Perturbation Size"
     ] = "Baseline (0.0)"
 
-    return results_df, fig, all(results_df["Passed"])
+    return (
+        results_df,
+        fig,
+        all(results_df["Passed"]),
+    )

validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py

@@ -229,5 +229,10 @@ def SHAPGlobalImportance(
     return (
         generate_shap_plot("mean", shap_values, shap_sample),
         generate_shap_plot("summary", shap_values, shap_sample),
-        RawData(shap_values=shap_values, shap_sample=shap_sample),
+        RawData(
+            shap_values=shap_values,
+            shap_sample=shap_sample,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.py

@@ -5,7 +5,7 @@
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -128,4 +128,12 @@ def ScoreProbabilityAlignment(
         height=600,
     )
 
-    return results_df, fig
+    # Include raw data for post-processing
+    raw_data = RawData(
+        score_bins=df[["score_bin", score_column]],
+        predicted_probabilities=df["probability"],
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
+
+    return results_df, fig, raw_data

validmind/tests/model_validation/sklearn/SilhouettePlot.py

@@ -110,5 +110,9 @@ def SilhouettePlot(model: VMModel, dataset: VMDataset):
             "Silhouette Score": silhouette_avg,
         },
         fig,
-        RawData(sample_silhouette_values=sample_silhouette_values),
+        RawData(
+            sample_silhouette_values=sample_silhouette_values,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/model_validation/sklearn/VMeasure.py

@@ -4,7 +4,7 @@
 
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -48,11 +48,14 @@ def VMeasure(dataset: VMDataset, model: VMModel):
     the other. The V Measure Score does not provide flexibility in assigning different weights to homogeneity and
     completeness.
     """
-    return [
-        {
-            "V Measure": metrics.v_measure_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
-            )
-        }
-    ]
+    v_measure = metrics.v_measure_score(
+        labels_true=dataset.y,
+        labels_pred=dataset.y_pred(model),
+    )
+
+    return (
+        [{"V Measure": v_measure}],
+        RawData(
+            v_measure_score=v_measure, model=model.input_id, dataset=dataset.input_id
+        ),
+    )

validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py

@@ -64,7 +64,11 @@ def CumulativePredictionProbabilities(dataset, model, title="Cumulative Probabil
 
     fig, fig_data = _plot_cumulative_prob(df, dataset.target_column, title)
 
-    return fig, RawData(cumulative_probabilities=fig_data)
+    return fig, RawData(
+        cumulative_probabilities=fig_data,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
 
 
 def _plot_cumulative_prob(df, target_col, title):

validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py

@@ -5,7 +5,7 @@
 import pandas as pd
 from statsmodels.stats.stattools import durbin_watson
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tasks("regression")
@@ -81,4 +81,9 @@ def DurbinWatsonTest(dataset, model, threshold=[1.5, 2.5]):
             "threshold": [str(threshold)],
             "autocorrelation": [get_autocorrelation(dw_statistic, threshold)],
         }
+    ), RawData(
+        residuals=residuals,
+        dw_statistic=dw_statistic,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )

validmind/tests/model_validation/statsmodels/GINITable.py

@@ -6,7 +6,7 @@ import numpy as np
 import pandas as pd
 from sklearn.metrics import roc_auc_score, roc_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("model_performance")
@@ -76,4 +76,11 @@ def GINITable(dataset, model):
             "GINI": [gini],
             "KS": [max(tpr - fpr)],
         }
+    ), RawData(
+        fpr=fpr,
+        tpr=tpr,
+        y_true=y_true,
+        y_prob=y_prob,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )

validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py

@@ -4,7 +4,7 @@
 
 from statsmodels.stats.diagnostic import kstest_normal
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import InvalidTestParametersError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -66,4 +66,4 @@ def KolmogorovSmirnov(model: VMModel, dataset: VMDataset, dist: str = "norm"):
             "P-Value": result["pvalue"],
         }
         for k, result in ks_values.items()
-    ]
+    ], RawData(ks_values=ks_values, dataset=dataset.input_id)

validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py

@@ -6,7 +6,7 @@
 import plotly.graph_objects as go
 from matplotlib import cm
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization", "credit_risk")
@@ -60,7 +60,11 @@ def PredictionProbabilitiesHistogram(
 
     fig = _plot_prob_histogram(df, dataset.target_column, title)
 
-    return fig
+    return fig, RawData(
+        probabilities=df["probabilities"],
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
 
 
 def _plot_prob_histogram(df, target_col, title):

validmind/tests/model_validation/statsmodels/RegressionCoeffs.py

@@ -7,7 +7,7 @@ import pandas as pd
 import plotly.graph_objects as go
 from scipy import stats
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 
 
@@ -97,4 +97,10 @@ def RegressionCoeffs(model):
         yaxis_title="Coefficients",
     )
 
-    return fig, coefficients
+    return (
+        fig,
+        coefficients,
+        RawData(
+            model=model.input_id, std_err=std_err, lower_ci=lower_ci, upper_ci=upper_ci
+        ),
+    )

validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py

@@ -90,4 +90,6 @@ def RegressionFeatureSignificance(
 
     plt.close()
 
-    return fig, RawData(coefficients=coefficients, pvalues=pvalues)
+    return fig, RawData(
+        coefficients=coefficients, pvalues=pvalues, model=model.input_id
+    )

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py

@@ -7,7 +7,7 @@ from typing import Union
 import matplotlib.pyplot as plt
 import pandas as pd
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
@@ -87,4 +87,9 @@ def RegressionModelForecastPlot(
 
     plt.close()
 
-    return fig
+    return fig, RawData(
+        observed_values=dataset.y.tolist(),
+        forecast_values=dataset.y_pred(model).tolist(),
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py

@@ -94,4 +94,6 @@ def RegressionModelForecastPlotLevels(
     return fig, RawData(
         y_transformed=dataset_y_transformed,
         y_pred_transformed=y_pred_transformed,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )