validmind 2.8.10__py3-none-any.whl → 2.8.20__py3-none-any.whl

validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py

@@ -6,7 +6,7 @@ import numpy as np
 from sklearn.metrics import roc_auc_score
 from sklearn.preprocessing import LabelBinarizer
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -62,19 +62,34 @@ def MinimumROCAUCScore(dataset: VMDataset, model: VMModel, min_threshold: float
         lb = LabelBinarizer()
         lb.fit(y_true)
 
+        y_true_binary = lb.transform(y_true)
+        y_score_binary = lb.transform(dataset.y_pred(model))
+
         roc_auc = roc_auc_score(
-            y_true=lb.transform(y_true),
-            y_score=lb.transform(dataset.y_pred(model)),
+            y_true=y_true_binary,
+            y_score=y_score_binary,
             average="macro",
         )
 
     else:
-        roc_auc = roc_auc_score(y_true=y_true, y_score=dataset.y_prob(model))
+        y_score_prob = dataset.y_prob(model)
+        roc_auc = roc_auc_score(y_true=y_true, y_score=y_score_prob)
 
-    return [
+    results = [
         {
             "Score": roc_auc,
             "Threshold": min_threshold,
             "Pass/Fail": "Pass" if roc_auc > min_threshold else "Fail",
         }
-    ], roc_auc > min_threshold
+    ]
+
+    return (
+        results,
+        roc_auc > min_threshold,
+        RawData(
+            y_true=y_true,
+            roc_auc=roc_auc,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
+    )

validmind/tests/model_validation/sklearn/OverfitDiagnosis.py

@@ -10,7 +10,7 @@ import pandas as pd
 import seaborn as sns
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
@@ -73,6 +73,7 @@ def _prepare_results(
         columns={"shape": "training records", f"{metric}": f"training {metric}"},
         inplace=True,
     )
+    results["test records"] = results_test["shape"]
     results[f"test {metric}"] = results_test[metric]
 
     # Adjust gap calculation based on metric directionality
@@ -292,11 +293,18 @@ def OverfitDiagnosis(
                 {
                     "Feature": feature_column,
                     "Slice": row["slice"],
-                    "Number of Records": row["training records"],
+                    "Number of Training Records": row["training records"],
+                    "Number of Test Records": row["test records"],
                     f"Training {metric.upper()}": row[f"training {metric}"],
                     f"Test {metric.upper()}": row[f"test {metric}"],
                     "Gap": row["gap"],
                 }
             )
 
-    return ({"Overfit Diagnosis": test_results}, *figures)
+    return (
+        {"Overfit Diagnosis": test_results},
+        *figures,
+        RawData(
+            model=model.input_id, datasets=[dataset.input_id for dataset in datasets]
+        ),
+    )

validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py

@@ -111,4 +111,8 @@ def PermutationFeatureImportance(
         height=figure_height,
     )
 
-    return fig, RawData(permutation_importance=pfi_values)
+    return fig, RawData(
+        permutation_importance=pfi_values,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py

@@ -209,5 +209,9 @@ def PopulationStabilityIndex(
             ],
         },
         fig,
-        RawData(psi_raw=psi_results),
+        RawData(
+            psi_raw=psi_results,
+            model=model.input_id,
+            datasets=[datasets[0].input_id, datasets[1].input_id],
+        ),
     )

validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py

@@ -83,4 +83,9 @@ def PrecisionRecallCurve(model: VMModel, dataset: VMDataset):
         ),
     )
 
-    return fig, RawData(precision=precision, recall=recall)
+    return fig, RawData(
+        precision=precision,
+        recall=recall,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/sklearn/ROCCurve.py

@@ -103,5 +103,7 @@ def ROCCurve(model: VMModel, dataset: VMDataset):
                 height=500,
             ),
         ),
-        RawData(fpr=fpr, tpr=tpr, auc=auc),
+        RawData(
+            fpr=fpr, tpr=tpr, auc=auc, model=model.input_id, dataset=dataset.input_id
+        ),
     )

validmind/tests/model_validation/sklearn/RegressionErrors.py

@@ -6,7 +6,7 @@ import numpy as np
 import pandas as pd
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("sklearn", "model_performance")
@@ -62,7 +62,11 @@ def RegressionErrors(model, dataset):
     y_pred = dataset.y_pred(model)
     y_true = y_true.astype(y_pred.dtype)
 
-    return _regression_errors(y_true, y_pred)
+    results_df = _regression_errors(y_true, y_pred)
+
+    return results_df, RawData(
+        y_true=y_true, y_pred=y_pred, model=model.input_id, dataset=dataset.input_id
+    )
 
 
 def _regression_errors(y_true, y_pred):

validmind/tests/model_validation/sklearn/RegressionPerformance.py

@@ -5,7 +5,7 @@
 import numpy as np
 from sklearn.metrics import mean_absolute_error, mean_squared_error
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
@@ -74,10 +74,15 @@ def RegressionPerformance(model: VMModel, dataset: VMDataset):
     # MBD calculation
     metrics["Mean Bias Deviation (MBD)"] = np.mean(y_pred - y_true)
 
-    return [
-        {
-            "Metric": metric,
-            "Value": value,
-        }
-        for metric, value in metrics.items()
-    ]
+    return (
+        [
+            {
+                "Metric": metric,
+                "Value": value,
+            }
+            for metric, value in metrics.items()
+        ],
+        RawData(
+            y_true=y_true, y_pred=y_pred, model=model.input_id, dataset=dataset.input_id
+        ),
+    )

validmind/tests/model_validation/sklearn/RegressionR2Square.py

@@ -5,7 +5,7 @@
 import pandas as pd
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.tests.model_validation.statsmodels.statsutils import adj_r2_score
 
 
@@ -55,11 +55,14 @@ def RegressionR2Square(dataset, model):
     y_pred = dataset.y_pred(model)
     y_true = y_true.astype(y_pred.dtype)
 
+    r2 = metrics.r2_score(y_true, y_pred)
+    adj_r2 = adj_r2_score(y_true, y_pred, len(y_true), len(dataset.feature_columns))
+
     return pd.DataFrame(
         {
-            "R-squared (R2) Score": [metrics.r2_score(y_true, y_pred)],
-            "Adjusted R-squared (R2) Score": [
-                adj_r2_score(y_true, y_pred, len(y_true), len(dataset.feature_columns))
-            ],
+            "R-squared (R2) Score": [r2],
+            "Adjusted R-squared (R2) Score": [adj_r2],
         }
+    ), RawData(
+        y_true=y_true, y_pred=y_pred, model=model.input_id, dataset=dataset.input_id
     )

validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py

@@ -327,4 +327,8 @@ def RobustnessDiagnosis(
         results_df["Perturbation Size"] == 0.0, "Perturbation Size"
     ] = "Baseline (0.0)"
 
-    return results_df, fig, all(results_df["Passed"])
+    return (
+        results_df,
+        fig,
+        all(results_df["Passed"]),
+    )

validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py

@@ -3,10 +3,12 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import warnings
+from typing import Dict, List, Optional, Union
 from warnings import filters as _warnings_filters
 
 import matplotlib.pyplot as plt
 import numpy as np
+import pandas as pd
 import shap
 
 from validmind import RawData, tags, tasks
@@ -18,7 +20,10 @@ from validmind.vm_models import VMDataset, VMModel
 logger = get_logger(__name__)
 
 
-def select_shap_values(shap_values, class_of_interest):
+def select_shap_values(
+    shap_values: Union[np.ndarray, List[np.ndarray]],
+    class_of_interest: Optional[int] = None,
+) -> np.ndarray:
     """Selects SHAP values for binary or multiclass classification.
 
     For regression models, returns the SHAP values directly as there are no classes.
@@ -41,32 +46,30 @@ def select_shap_values(shap_values, class_of_interest):
     """
     if not isinstance(shap_values, list):
         # For regression, return the SHAP values as they are
-        # TODO: shap_values is always an array of all predictions, how is the if above supposed to work?
-        # logger.info("Returning SHAP values as-is.")
-        return shap_values
-
-    num_classes = len(shap_values)
-
-    # Default to class 1 for binary classification where no class is specified
-    if num_classes == 2 and class_of_interest is None:
-        logger.debug("Using SHAP values for class 1 (positive class).")
-        return shap_values[1]
+        selected_values = shap_values
+    else:
+        num_classes = len(shap_values)
+        # Default to class 1 for binary classification where no class is specified
+        if num_classes == 2 and class_of_interest is None:
+            selected_values = shap_values[1]
+        # Otherwise, use the specified class_of_interest
+        elif class_of_interest is not None and 0 <= class_of_interest < num_classes:
+            selected_values = shap_values[class_of_interest]
+        else:
+            raise ValueError(
+                f"Invalid class_of_interest: {class_of_interest}. Must be between 0 and {num_classes - 1}."
+            )
 
-    # Otherwise, use the specified class_of_interest
-    if (
-        class_of_interest is None
-        or class_of_interest < 0
-        or class_of_interest >= num_classes
-    ):
-        raise ValueError(
-            f"Invalid class_of_interest: {class_of_interest}. Must be between 0 and {num_classes - 1}."
-        )
+    # Add type conversion here to ensure proper float array
+    if hasattr(selected_values, "dtype"):
+        selected_values = np.array(selected_values, dtype=np.float64)
 
-    logger.debug(f"Using SHAP values for class {class_of_interest}.")
-    return shap_values[class_of_interest]
+    return selected_values
 
 
-def generate_shap_plot(type_, shap_values, x_test):
+def generate_shap_plot(
+    type_: str, shap_values: np.ndarray, x_test: Union[np.ndarray, pd.DataFrame]
+) -> plt.Figure:
     """Plots two types of SHAP global importance (SHAP).
 
     Args:
@@ -117,8 +120,8 @@ def SHAPGlobalImportance(
     dataset: VMDataset,
     kernel_explainer_samples: int = 10,
     tree_or_linear_explainer_samples: int = 200,
-    class_of_interest: int = None,
-):
+    class_of_interest: Optional[int] = None,
+) -> Dict[str, Union[plt.Figure, Dict[str, float]]]:
     """
     Evaluates and visualizes global feature importance using SHAP values for model explanation and risk identification.
 
@@ -229,5 +232,10 @@ def SHAPGlobalImportance(
     return (
         generate_shap_plot("mean", shap_values, shap_sample),
         generate_shap_plot("summary", shap_values, shap_sample),
-        RawData(shap_values=shap_values, shap_sample=shap_sample),
+        RawData(
+            shap_values=shap_values,
+            shap_sample=shap_sample,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.py

@@ -5,7 +5,7 @@
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -128,4 +128,12 @@ def ScoreProbabilityAlignment(
         height=600,
     )
 
-    return results_df, fig
+    # Include raw data for post-processing
+    raw_data = RawData(
+        score_bins=df[["score_bin", score_column]],
+        predicted_probabilities=df["probability"],
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
+
+    return results_df, fig, raw_data

validmind/tests/model_validation/sklearn/SilhouettePlot.py

@@ -110,5 +110,9 @@ def SilhouettePlot(model: VMModel, dataset: VMDataset):
             "Silhouette Score": silhouette_avg,
         },
         fig,
-        RawData(sample_silhouette_values=sample_silhouette_values),
+        RawData(
+            sample_silhouette_values=sample_silhouette_values,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/model_validation/sklearn/VMeasure.py

@@ -4,7 +4,7 @@
 
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -48,11 +48,14 @@ def VMeasure(dataset: VMDataset, model: VMModel):
     the other. The V Measure Score does not provide flexibility in assigning different weights to homogeneity and
     completeness.
     """
-    return [
-        {
-            "V Measure": metrics.v_measure_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
-            )
-        }
-    ]
+    v_measure = metrics.v_measure_score(
+        labels_true=dataset.y,
+        labels_pred=dataset.y_pred(model),
+    )
+
+    return (
+        [{"V Measure": v_measure}],
+        RawData(
+            v_measure_score=v_measure, model=model.input_id, dataset=dataset.input_id
+        ),
+    )

validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py

@@ -47,7 +47,7 @@ def _compute_metrics(
         None: The computed metrics are appended to the `results` dictionary in-place.
     """
     results["Slice"].append(str(region))
-    results["Shape"].append(df_region.shape[0])
+    results["Number of Records"].append(df_region.shape[0])
     results["Feature"].append(feature_column)
 
     # Check if df_region is an empty dataframe and if so, append 0 to all metrics
@@ -222,7 +222,7 @@ def WeakspotsDiagnosis(
     thresholds = thresholds or DEFAULT_THRESHOLDS
     thresholds = {k.title(): v for k, v in thresholds.items()}
 
-    results_headers = ["Slice", "Shape", "Feature"]
+    results_headers = ["Slice", "Number of Records", "Feature"]
     results_headers.extend(metrics.keys())
 
     figures = []
@@ -236,19 +236,20 @@ def WeakspotsDiagnosis(
         feature_columns
         + [datasets[1].target_column, datasets[1].prediction_column(model)]
     ]
-
+    results_1 = pd.DataFrame()
+    results_2 = pd.DataFrame()
     for feature in feature_columns:
         bins = 10
         if feature in datasets[0].feature_columns_categorical:
             bins = len(df_1[feature].unique())
         df_1["bin"] = pd.cut(df_1[feature], bins=bins)
 
-        results_1 = {k: [] for k in results_headers}
-        results_2 = {k: [] for k in results_headers}
+        r1 = {k: [] for k in results_headers}
+        r2 = {k: [] for k in results_headers}
 
         for region, df_region in df_1.groupby("bin"):
             _compute_metrics(
-                results=results_1,
+                results=r1,
                 metrics=metrics,
                 region=region,
                 df_region=df_region,
@@ -260,7 +261,7 @@ def WeakspotsDiagnosis(
                 (df_2[feature] > region.left) & (df_2[feature] <= region.right)
             ]
             _compute_metrics(
-                results=results_2,
+                results=r2,
                 metrics=metrics,
                 region=region,
                 df_region=df_2_region,
@@ -271,8 +272,8 @@ def WeakspotsDiagnosis(
 
         for metric in metrics.keys():
             fig, df = _plot_weak_spots(
-                results_1=results_1,
-                results_2=results_2,
+                results_1=r1,
+                results_2=r2,
                 feature_column=feature,
                 metric=metric,
                 threshold=thresholds[metric],
@@ -284,6 +285,8 @@ def WeakspotsDiagnosis(
         # rely on visual assessment for this test for now.
         if not df[df[list(thresholds.keys())].lt(thresholds).any(axis=1)].empty:
             passed = False
+        results_1 = pd.concat([results_1, pd.DataFrame(r1)])
+        results_2 = pd.concat([results_2, pd.DataFrame(r2)])
 
     return (
         pd.concat(
@@ -291,7 +294,9 @@ def WeakspotsDiagnosis(
                 pd.DataFrame(results_1).assign(Dataset=datasets[0].input_id),
                 pd.DataFrame(results_2).assign(Dataset=datasets[1].input_id),
             ]
-        ).sort_values(["Feature", "Dataset"]),
+        )
+        .reset_index(drop=True)
+        .sort_values(["Feature", "Dataset"]),
         *figures,
         passed,
     )

validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py

@@ -64,7 +64,11 @@ def CumulativePredictionProbabilities(dataset, model, title="Cumulative Probabil
 
     fig, fig_data = _plot_cumulative_prob(df, dataset.target_column, title)
 
-    return fig, RawData(cumulative_probabilities=fig_data)
+    return fig, RawData(
+        cumulative_probabilities=fig_data,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
 
 
 def _plot_cumulative_prob(df, target_col, title):

validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py

@@ -5,7 +5,7 @@
 import pandas as pd
 from statsmodels.stats.stattools import durbin_watson
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tasks("regression")
@@ -81,4 +81,9 @@ def DurbinWatsonTest(dataset, model, threshold=[1.5, 2.5]):
             "threshold": [str(threshold)],
             "autocorrelation": [get_autocorrelation(dw_statistic, threshold)],
         }
+    ), RawData(
+        residuals=residuals,
+        dw_statistic=dw_statistic,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )

validmind/tests/model_validation/statsmodels/GINITable.py

@@ -6,7 +6,7 @@ import numpy as np
 import pandas as pd
 from sklearn.metrics import roc_auc_score, roc_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("model_performance")
@@ -76,4 +76,11 @@ def GINITable(dataset, model):
             "GINI": [gini],
             "KS": [max(tpr - fpr)],
         }
+    ), RawData(
+        fpr=fpr,
+        tpr=tpr,
+        y_true=y_true,
+        y_prob=y_prob,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )

validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py

@@ -4,7 +4,7 @@
 
 from statsmodels.stats.diagnostic import kstest_normal
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import InvalidTestParametersError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -66,4 +66,4 @@ def KolmogorovSmirnov(model: VMModel, dataset: VMDataset, dist: str = "norm"):
             "P-Value": result["pvalue"],
         }
         for k, result in ks_values.items()
-    ]
+    ], RawData(ks_values=ks_values, dataset=dataset.input_id)

validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py

@@ -6,7 +6,7 @@
 import plotly.graph_objects as go
 from matplotlib import cm
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization", "credit_risk")
@@ -60,7 +60,11 @@ def PredictionProbabilitiesHistogram(
 
     fig = _plot_prob_histogram(df, dataset.target_column, title)
 
-    return fig
+    return fig, RawData(
+        probabilities=df["probabilities"],
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
 
 
 def _plot_prob_histogram(df, target_col, title):

validmind/tests/model_validation/statsmodels/RegressionCoeffs.py

@@ -7,7 +7,7 @@ import pandas as pd
 import plotly.graph_objects as go
 from scipy import stats
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 
 
@@ -97,4 +97,10 @@ def RegressionCoeffs(model):
         yaxis_title="Coefficients",
     )
 
-    return fig, coefficients
+    return (
+        fig,
+        coefficients,
+        RawData(
+            model=model.input_id, std_err=std_err, lower_ci=lower_ci, upper_ci=upper_ci
+        ),
+    )

validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py

@@ -90,4 +90,6 @@ def RegressionFeatureSignificance(
 
     plt.close()
 
-    return fig, RawData(coefficients=coefficients, pvalues=pvalues)
+    return fig, RawData(
+        coefficients=coefficients, pvalues=pvalues, model=model.input_id
+    )

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py

@@ -7,7 +7,7 @@ from typing import Union
 import matplotlib.pyplot as plt
 import pandas as pd
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
@@ -87,4 +87,9 @@ def RegressionModelForecastPlot(
 
     plt.close()
 
-    return fig
+    return fig, RawData(
+        observed_values=dataset.y.tolist(),
+        forecast_values=dataset.y_pred(model).tolist(),
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py

@@ -94,4 +94,6 @@ def RegressionModelForecastPlotLevels(
     return fig, RawData(
         y_transformed=dataset_y_transformed,
         y_pred_transformed=y_pred_transformed,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )

validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py

@@ -114,4 +114,6 @@ def RegressionModelSensitivityPlot(
     return fig, RawData(
         transformed_target=transformed_target,
         transformed_predictions=transformed_predictions,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )

validmind/tests/model_validation/statsmodels/RegressionModelSummary.py

@@ -4,7 +4,7 @@
 
 from sklearn.metrics import mean_squared_error, r2_score
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 from .statsutils import adj_r2_score
@@ -58,4 +58,6 @@ def RegressionModelSummary(dataset: VMDataset, model: VMModel):
             "MSE": mean_squared_error(y_true, y_pred, squared=True),
             "RMSE": mean_squared_error(y_true, y_pred, squared=False),
         }
-    ]
+    ], RawData(
+        y_true=y_true, y_pred=y_pred, model=model.input_id, dataset=dataset.input_id
+    )

validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py

@@ -97,4 +97,6 @@ def RegressionPermutationFeatureImportance(
         height=figure_height,
     )
 
-    return fig, RawData(importances=importances)
+    return fig, RawData(
+        importances=importances, model=model.input_id, dataset=dataset.input_id
+    )