validmind 2.7.12__py3-none-any.whl → 2.8.12__py3-none-any.whl

validmind/tests/model_validation/sklearn/OverfitDiagnosis.py

@@ -10,7 +10,7 @@ import pandas as pd
 import seaborn as sns
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
@@ -299,4 +299,10 @@ def OverfitDiagnosis(
                 }
             )
 
-    return ({"Overfit Diagnosis": test_results}, *figures)
+    return (
+        {"Overfit Diagnosis": test_results},
+        *figures,
+        RawData(
+            model=model.input_id, datasets=[dataset.input_id for dataset in datasets]
+        ),
+    )

validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py

@@ -111,4 +111,8 @@ def PermutationFeatureImportance(
         height=figure_height,
     )
 
-    return fig, RawData(permutation_importance=pfi_values)
+    return fig, RawData(
+        permutation_importance=pfi_values,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py

@@ -209,5 +209,9 @@ def PopulationStabilityIndex(
             ],
         },
         fig,
-        RawData(psi_raw=psi_results),
+        RawData(
+            psi_raw=psi_results,
+            model=model.input_id,
+            datasets=[datasets[0].input_id, datasets[1].input_id],
+        ),
     )

validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py

@@ -83,4 +83,9 @@ def PrecisionRecallCurve(model: VMModel, dataset: VMDataset):
         ),
     )
 
-    return fig, RawData(precision=precision, recall=recall)
+    return fig, RawData(
+        precision=precision,
+        recall=recall,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/sklearn/ROCCurve.py

@@ -103,5 +103,7 @@ def ROCCurve(model: VMModel, dataset: VMDataset):
                 height=500,
             ),
         ),
-        RawData(fpr=fpr, tpr=tpr, auc=auc),
+        RawData(
+            fpr=fpr, tpr=tpr, auc=auc, model=model.input_id, dataset=dataset.input_id
+        ),
     )

validmind/tests/model_validation/sklearn/RegressionErrors.py

@@ -6,7 +6,7 @@ import numpy as np
 import pandas as pd
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("sklearn", "model_performance")
@@ -62,7 +62,11 @@ def RegressionErrors(model, dataset):
     y_pred = dataset.y_pred(model)
     y_true = y_true.astype(y_pred.dtype)
 
-    return _regression_errors(y_true, y_pred)
+    results_df = _regression_errors(y_true, y_pred)
+
+    return results_df, RawData(
+        y_true=y_true, y_pred=y_pred, model=model.input_id, dataset=dataset.input_id
+    )
 
 
 def _regression_errors(y_true, y_pred):

validmind/tests/model_validation/sklearn/RegressionPerformance.py

@@ -5,7 +5,7 @@
 import numpy as np
 from sklearn.metrics import mean_absolute_error, mean_squared_error
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
@@ -74,10 +74,15 @@ def RegressionPerformance(model: VMModel, dataset: VMDataset):
     # MBD calculation
     metrics["Mean Bias Deviation (MBD)"] = np.mean(y_pred - y_true)
 
-    return [
-        {
-            "Metric": metric,
-            "Value": value,
-        }
-        for metric, value in metrics.items()
-    ]
+    return (
+        [
+            {
+                "Metric": metric,
+                "Value": value,
+            }
+            for metric, value in metrics.items()
+        ],
+        RawData(
+            y_true=y_true, y_pred=y_pred, model=model.input_id, dataset=dataset.input_id
+        ),
+    )

validmind/tests/model_validation/sklearn/RegressionR2Square.py

@@ -5,7 +5,7 @@
 import pandas as pd
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.tests.model_validation.statsmodels.statsutils import adj_r2_score
 
 
@@ -55,11 +55,14 @@ def RegressionR2Square(dataset, model):
     y_pred = dataset.y_pred(model)
     y_true = y_true.astype(y_pred.dtype)
 
+    r2 = metrics.r2_score(y_true, y_pred)
+    adj_r2 = adj_r2_score(y_true, y_pred, len(y_true), len(dataset.feature_columns))
+
     return pd.DataFrame(
         {
-            "R-squared (R2) Score": [metrics.r2_score(y_true, y_pred)],
-            "Adjusted R-squared (R2) Score": [
-                adj_r2_score(y_true, y_pred, len(y_true), len(dataset.feature_columns))
-            ],
+            "R-squared (R2) Score": [r2],
+            "Adjusted R-squared (R2) Score": [adj_r2],
         }
+    ), RawData(
+        y_true=y_true, y_pred=y_pred, model=model.input_id, dataset=dataset.input_id
     )

validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py

@@ -327,4 +327,8 @@ def RobustnessDiagnosis(
         results_df["Perturbation Size"] == 0.0, "Perturbation Size"
     ] = "Baseline (0.0)"
 
-    return results_df, fig, all(results_df["Passed"])
+    return (
+        results_df,
+        fig,
+        all(results_df["Passed"]),
+    )

validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py

@@ -229,5 +229,10 @@ def SHAPGlobalImportance(
     return (
         generate_shap_plot("mean", shap_values, shap_sample),
         generate_shap_plot("summary", shap_values, shap_sample),
-        RawData(shap_values=shap_values, shap_sample=shap_sample),
+        RawData(
+            shap_values=shap_values,
+            shap_sample=shap_sample,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.py

@@ -5,7 +5,7 @@
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -128,4 +128,12 @@ def ScoreProbabilityAlignment(
         height=600,
     )
 
-    return results_df, fig
+    # Include raw data for post-processing
+    raw_data = RawData(
+        score_bins=df[["score_bin", score_column]],
+        predicted_probabilities=df["probability"],
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
+
+    return results_df, fig, raw_data

validmind/tests/model_validation/sklearn/SilhouettePlot.py

@@ -110,5 +110,9 @@ def SilhouettePlot(model: VMModel, dataset: VMDataset):
             "Silhouette Score": silhouette_avg,
         },
         fig,
-        RawData(sample_silhouette_values=sample_silhouette_values),
+        RawData(
+            sample_silhouette_values=sample_silhouette_values,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/model_validation/sklearn/VMeasure.py

@@ -4,7 +4,7 @@
 
 from sklearn import metrics
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -48,11 +48,14 @@ def VMeasure(dataset: VMDataset, model: VMModel):
     the other. The V Measure Score does not provide flexibility in assigning different weights to homogeneity and
     completeness.
     """
-    return [
-        {
-            "V Measure": metrics.v_measure_score(
-                labels_true=dataset.y,
-                labels_pred=dataset.y_pred(model),
-            )
-        }
-    ]
+    v_measure = metrics.v_measure_score(
+        labels_true=dataset.y,
+        labels_pred=dataset.y_pred(model),
+    )
+
+    return (
+        [{"V Measure": v_measure}],
+        RawData(
+            v_measure_score=v_measure, model=model.input_id, dataset=dataset.input_id
+        ),
+    )

validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py

@@ -64,7 +64,11 @@ def CumulativePredictionProbabilities(dataset, model, title="Cumulative Probabil
 
     fig, fig_data = _plot_cumulative_prob(df, dataset.target_column, title)
 
-    return fig, RawData(cumulative_probabilities=fig_data)
+    return fig, RawData(
+        cumulative_probabilities=fig_data,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
 
 
 def _plot_cumulative_prob(df, target_col, title):

validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py

@@ -5,7 +5,7 @@
 import pandas as pd
 from statsmodels.stats.stattools import durbin_watson
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tasks("regression")
@@ -81,4 +81,9 @@ def DurbinWatsonTest(dataset, model, threshold=[1.5, 2.5]):
             "threshold": [str(threshold)],
             "autocorrelation": [get_autocorrelation(dw_statistic, threshold)],
         }
+    ), RawData(
+        residuals=residuals,
+        dw_statistic=dw_statistic,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )

validmind/tests/model_validation/statsmodels/GINITable.py

@@ -6,7 +6,7 @@ import numpy as np
 import pandas as pd
 from sklearn.metrics import roc_auc_score, roc_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("model_performance")
@@ -76,4 +76,11 @@ def GINITable(dataset, model):
             "GINI": [gini],
             "KS": [max(tpr - fpr)],
         }
+    ), RawData(
+        fpr=fpr,
+        tpr=tpr,
+        y_true=y_true,
+        y_prob=y_prob,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )

validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py

@@ -4,7 +4,7 @@
 
 from statsmodels.stats.diagnostic import kstest_normal
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import InvalidTestParametersError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -66,4 +66,4 @@ def KolmogorovSmirnov(model: VMModel, dataset: VMDataset, dist: str = "norm"):
             "P-Value": result["pvalue"],
         }
         for k, result in ks_values.items()
-    ]
+    ], RawData(ks_values=ks_values, dataset=dataset.input_id)

validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py

@@ -6,7 +6,7 @@
 import plotly.graph_objects as go
 from matplotlib import cm
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization", "credit_risk")
@@ -60,7 +60,11 @@ def PredictionProbabilitiesHistogram(
 
     fig = _plot_prob_histogram(df, dataset.target_column, title)
 
-    return fig
+    return fig, RawData(
+        probabilities=df["probabilities"],
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
 
 
 def _plot_prob_histogram(df, target_col, title):

validmind/tests/model_validation/statsmodels/RegressionCoeffs.py

@@ -7,7 +7,7 @@ import pandas as pd
 import plotly.graph_objects as go
 from scipy import stats
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 
 
@@ -97,4 +97,10 @@ def RegressionCoeffs(model):
         yaxis_title="Coefficients",
     )
 
-    return fig, coefficients
+    return (
+        fig,
+        coefficients,
+        RawData(
+            model=model.input_id, std_err=std_err, lower_ci=lower_ci, upper_ci=upper_ci
+        ),
+    )

validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py

@@ -90,4 +90,6 @@ def RegressionFeatureSignificance(
 
     plt.close()
 
-    return fig, RawData(coefficients=coefficients, pvalues=pvalues)
+    return fig, RawData(
+        coefficients=coefficients, pvalues=pvalues, model=model.input_id
+    )

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py

@@ -7,7 +7,7 @@ from typing import Union
 import matplotlib.pyplot as plt
 import pandas as pd
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
@@ -87,4 +87,9 @@ def RegressionModelForecastPlot(
 
     plt.close()
 
-    return fig
+    return fig, RawData(
+        observed_values=dataset.y.tolist(),
+        forecast_values=dataset.y_pred(model).tolist(),
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )

validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py

@@ -94,4 +94,6 @@ def RegressionModelForecastPlotLevels(
     return fig, RawData(
         y_transformed=dataset_y_transformed,
         y_pred_transformed=y_pred_transformed,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )

validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py

@@ -114,4 +114,6 @@ def RegressionModelSensitivityPlot(
     return fig, RawData(
         transformed_target=transformed_target,
         transformed_predictions=transformed_predictions,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )

validmind/tests/model_validation/statsmodels/RegressionModelSummary.py

@@ -4,7 +4,7 @@
 
 from sklearn.metrics import mean_squared_error, r2_score
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 from .statsutils import adj_r2_score
@@ -45,17 +45,19 @@ def RegressionModelSummary(dataset: VMDataset, model: VMModel):
     - A high R-Squared or Adjusted R-Squared may not necessarily indicate a good model, especially in cases of
     overfitting.
     """
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
     return [
         {
             "Independent Variables": dataset.feature_columns,
-            "R-Squared": r2_score(dataset.y, dataset.y_pred(model)),
+            "R-Squared": r2_score(y_true, y_pred),
             "Adjusted R-Squared": adj_r2_score(
-                dataset.y,
-                dataset.y_pred(model),
-                len(dataset.y),
-                len(dataset.feature_columns),
+                y_true, y_pred, len(y_true), len(dataset.feature_columns)
             ),
-            "MSE": mean_squared_error(dataset.y, dataset.y_pred(model), squared=True),
-            "RMSE": mean_squared_error(dataset.y, dataset.y_pred(model), squared=False),
+            "MSE": mean_squared_error(y_true, y_pred, squared=True),
+            "RMSE": mean_squared_error(y_true, y_pred, squared=False),
         }
-    ]
+    ], RawData(
+        y_true=y_true, y_pred=y_pred, model=model.input_id, dataset=dataset.input_id
+    )

validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py

@@ -97,4 +97,6 @@ def RegressionPermutationFeatureImportance(
         height=figure_height,
     )
 
-    return fig, RawData(importances=importances)
+    return fig, RawData(
+        importances=importances, model=model.input_id, dataset=dataset.input_id
+    )

validmind/tests/ongoing_monitoring/CalibrationCurveDrift.py

@@ -9,7 +9,7 @@ import pandas as pd
 import plotly.graph_objects as go
 from sklearn.calibration import calibration_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -217,4 +217,14 @@ def CalibrationCurveDrift(
         fig,
         {"Mean Predicted Probabilities": pred_df, "Fraction of Positives": true_df},
         pass_fail_bool,
+        RawData(
+            prob_true_ref=prob_true_ref,
+            prob_pred_ref=prob_pred_ref,
+            prob_true_mon=prob_true_mon,
+            prob_pred_mon=prob_pred_mon,
+            bin_labels=bin_labels,
+            model=model.input_id,
+            dataset_ref=datasets[0].input_id,
+            dataset_mon=datasets[1].input_id,
+        ),
     )

validmind/tests/ongoing_monitoring/ClassificationAccuracyDrift.py

@@ -8,7 +8,7 @@ import numpy as np
 import pandas as pd
 from sklearn.metrics import classification_report
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -145,4 +145,12 @@ def ClassificationAccuracyDrift(
     # Calculate overall pass/fail
     pass_fail_bool = (df["Pass/Fail"] == "Pass").all()
 
-    return ({"Classification Accuracy Metrics": df}, pass_fail_bool)
+    raw_data = RawData(
+        report_reference=report_ref,
+        report_monitoring=report_mon,
+        model=model.input_id,
+        dataset_reference=datasets[0].input_id,
+        dataset_monitoring=datasets[1].input_id,
+    )
+
+    return ({"Classification Accuracy Metrics": df}, pass_fail_bool, raw_data)

validmind/tests/ongoing_monitoring/ConfusionMatrixDrift.py

@@ -8,7 +8,7 @@ import numpy as np
 import pandas as pd
 from sklearn.metrics import confusion_matrix
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -190,4 +190,11 @@ def ConfusionMatrixDrift(
     return (
         {"Confusion Matrix Metrics": metrics_df, "Sample Counts": counts_df},
         pass_fail_bool,
+        RawData(
+            confusion_matrix_reference=cm_ref,
+            confusion_matrix_monitoring=cm_mon,
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
     )

validmind/tests/ongoing_monitoring/CumulativePredictionProbabilitiesDrift.py

@@ -8,7 +8,7 @@ import numpy as np
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -83,6 +83,7 @@ def CumulativePredictionProbabilitiesDrift(
     diff_color = "rgba(148, 103, 189, 0.8)"  # Purple with 0.8 opacity
 
     figures = []
+    raw_data = {}
     for class_value in classes:
         # Create figure with secondary y-axis
         fig = make_subplots(
@@ -175,4 +176,19 @@ def CumulativePredictionProbabilitiesDrift(
 
         figures.append(fig)
 
-    return tuple(figures)
+        # Store raw data for current class
+        raw_data[f"class_{class_value}_ref_probs"] = ref_probs
+        raw_data[f"class_{class_value}_mon_probs"] = mon_probs
+        raw_data[f"class_{class_value}_ref_sorted"] = ref_sorted
+        raw_data[f"class_{class_value}_ref_cumsum"] = ref_cumsum
+        raw_data[f"class_{class_value}_mon_sorted"] = mon_sorted
+        raw_data[f"class_{class_value}_mon_cumsum"] = mon_cumsum
+
+    return tuple(figures) + (
+        RawData(
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+            **raw_data,
+        ),
+    )

validmind/tests/ongoing_monitoring/FeatureDrift.py

@@ -6,7 +6,7 @@ import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 def calculate_psi_score(actual, expected):
@@ -183,4 +183,11 @@ def FeatureDrift(
     # Calculate overall pass/fail
     pass_fail_bool = (psi_df["Pass/Fail"] == "Pass").all()
 
-    return ({"PSI Scores": psi_df}, *figures, pass_fail_bool)
+    # Prepare raw data
+    raw_data = RawData(
+        distributions=distributions,
+        dataset_reference=datasets[0].input_id,
+        dataset_monitoring=datasets[1].input_id,
+    )
+
+    return ({"PSI Scores": psi_df}, *figures, pass_fail_bool, raw_data)

validmind/tests/ongoing_monitoring/PredictionAcrossEachFeature.py

@@ -5,7 +5,7 @@
 
 import matplotlib.pyplot as plt
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization")
@@ -74,4 +74,10 @@ def PredictionAcrossEachFeature(datasets, model):
         figures_to_save.append(fig)
         plt.close()
 
-    return tuple(figures_to_save)
+    return tuple(figures_to_save), RawData(
+        y_prob_reference=y_prob_reference,
+        y_prob_monitoring=y_prob_monitoring,
+        model=model.input_id,
+        dataset_reference=datasets[0].input_id,
+        dataset_monitoring=datasets[1].input_id,
+    )

validmind/tests/ongoing_monitoring/PredictionCorrelation.py

@@ -5,7 +5,7 @@
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization")
@@ -140,4 +140,15 @@ def PredictionCorrelation(datasets, model, drift_pct_threshold=20):
     # Calculate overall pass/fail
     pass_fail_bool = (corr_final["Pass/Fail"] == "Pass").all()
 
-    return ({"Correlation Pair Table": corr_final}, fig, pass_fail_bool)
+    return (
+        {"Correlation Pair Table": corr_final},
+        fig,
+        pass_fail_bool,
+        RawData(
+            reference_correlations=corr_ref.to_dict(),
+            monitoring_correlations=corr_mon.to_dict(),
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
+    )

validmind/tests/ongoing_monitoring/PredictionProbabilitiesHistogramDrift.py

@@ -10,7 +10,7 @@ import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 from scipy import stats
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -201,4 +201,15 @@ def PredictionProbabilitiesHistogramDrift(
             }
         )
 
-    return fig, tables, all_passed
+    return (
+        fig,
+        tables,
+        all_passed,
+        RawData(
+            reference_probabilities=y_prob_ref,
+            monitoring_probabilities=y_prob_mon,
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
+    )