validmind 2.8.10__py3-none-any.whl → 2.8.12__py3-none-any.whl

validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py

@@ -114,4 +114,6 @@ def RegressionModelSensitivityPlot(
     return fig, RawData(
         transformed_target=transformed_target,
         transformed_predictions=transformed_predictions,
+        model=model.input_id,
+        dataset=dataset.input_id,
     )

validmind/tests/model_validation/statsmodels/RegressionModelSummary.py

@@ -4,7 +4,7 @@
 
 from sklearn.metrics import mean_squared_error, r2_score
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 from .statsutils import adj_r2_score
@@ -58,4 +58,6 @@ def RegressionModelSummary(dataset: VMDataset, model: VMModel):
             "MSE": mean_squared_error(y_true, y_pred, squared=True),
             "RMSE": mean_squared_error(y_true, y_pred, squared=False),
         }
-    ]
+    ], RawData(
+        y_true=y_true, y_pred=y_pred, model=model.input_id, dataset=dataset.input_id
+    )

validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py

@@ -97,4 +97,6 @@ def RegressionPermutationFeatureImportance(
         height=figure_height,
     )
 
-    return fig, RawData(importances=importances)
+    return fig, RawData(
+        importances=importances, model=model.input_id, dataset=dataset.input_id
+    )

validmind/tests/ongoing_monitoring/CalibrationCurveDrift.py

@@ -9,7 +9,7 @@ import pandas as pd
 import plotly.graph_objects as go
 from sklearn.calibration import calibration_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -217,4 +217,14 @@ def CalibrationCurveDrift(
         fig,
         {"Mean Predicted Probabilities": pred_df, "Fraction of Positives": true_df},
         pass_fail_bool,
+        RawData(
+            prob_true_ref=prob_true_ref,
+            prob_pred_ref=prob_pred_ref,
+            prob_true_mon=prob_true_mon,
+            prob_pred_mon=prob_pred_mon,
+            bin_labels=bin_labels,
+            model=model.input_id,
+            dataset_ref=datasets[0].input_id,
+            dataset_mon=datasets[1].input_id,
+        ),
     )

validmind/tests/ongoing_monitoring/ClassificationAccuracyDrift.py

@@ -8,7 +8,7 @@ import numpy as np
 import pandas as pd
 from sklearn.metrics import classification_report
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -145,4 +145,12 @@ def ClassificationAccuracyDrift(
     # Calculate overall pass/fail
     pass_fail_bool = (df["Pass/Fail"] == "Pass").all()
 
-    return ({"Classification Accuracy Metrics": df}, pass_fail_bool)
+    raw_data = RawData(
+        report_reference=report_ref,
+        report_monitoring=report_mon,
+        model=model.input_id,
+        dataset_reference=datasets[0].input_id,
+        dataset_monitoring=datasets[1].input_id,
+    )
+
+    return ({"Classification Accuracy Metrics": df}, pass_fail_bool, raw_data)

validmind/tests/ongoing_monitoring/ConfusionMatrixDrift.py

@@ -8,7 +8,7 @@ import numpy as np
 import pandas as pd
 from sklearn.metrics import confusion_matrix
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -190,4 +190,11 @@ def ConfusionMatrixDrift(
     return (
         {"Confusion Matrix Metrics": metrics_df, "Sample Counts": counts_df},
         pass_fail_bool,
+        RawData(
+            confusion_matrix_reference=cm_ref,
+            confusion_matrix_monitoring=cm_mon,
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
     )

validmind/tests/ongoing_monitoring/CumulativePredictionProbabilitiesDrift.py

@@ -8,7 +8,7 @@ import numpy as np
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -83,6 +83,7 @@ def CumulativePredictionProbabilitiesDrift(
     diff_color = "rgba(148, 103, 189, 0.8)"  # Purple with 0.8 opacity
 
     figures = []
+    raw_data = {}
     for class_value in classes:
         # Create figure with secondary y-axis
         fig = make_subplots(
@@ -175,4 +176,19 @@ def CumulativePredictionProbabilitiesDrift(
 
         figures.append(fig)
 
-    return tuple(figures)
+        # Store raw data for current class
+        raw_data[f"class_{class_value}_ref_probs"] = ref_probs
+        raw_data[f"class_{class_value}_mon_probs"] = mon_probs
+        raw_data[f"class_{class_value}_ref_sorted"] = ref_sorted
+        raw_data[f"class_{class_value}_ref_cumsum"] = ref_cumsum
+        raw_data[f"class_{class_value}_mon_sorted"] = mon_sorted
+        raw_data[f"class_{class_value}_mon_cumsum"] = mon_cumsum
+
+    return tuple(figures) + (
+        RawData(
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+            **raw_data,
+        ),
+    )

validmind/tests/ongoing_monitoring/FeatureDrift.py

@@ -6,7 +6,7 @@ import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 def calculate_psi_score(actual, expected):
@@ -183,4 +183,11 @@ def FeatureDrift(
     # Calculate overall pass/fail
     pass_fail_bool = (psi_df["Pass/Fail"] == "Pass").all()
 
-    return ({"PSI Scores": psi_df}, *figures, pass_fail_bool)
+    # Prepare raw data
+    raw_data = RawData(
+        distributions=distributions,
+        dataset_reference=datasets[0].input_id,
+        dataset_monitoring=datasets[1].input_id,
+    )
+
+    return ({"PSI Scores": psi_df}, *figures, pass_fail_bool, raw_data)

validmind/tests/ongoing_monitoring/PredictionAcrossEachFeature.py

@@ -5,7 +5,7 @@
 
 import matplotlib.pyplot as plt
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization")
@@ -74,4 +74,10 @@ def PredictionAcrossEachFeature(datasets, model):
         figures_to_save.append(fig)
         plt.close()
 
-    return tuple(figures_to_save)
+    return tuple(figures_to_save), RawData(
+        y_prob_reference=y_prob_reference,
+        y_prob_monitoring=y_prob_monitoring,
+        model=model.input_id,
+        dataset_reference=datasets[0].input_id,
+        dataset_monitoring=datasets[1].input_id,
+    )

validmind/tests/ongoing_monitoring/PredictionCorrelation.py

@@ -5,7 +5,7 @@
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization")
@@ -140,4 +140,15 @@ def PredictionCorrelation(datasets, model, drift_pct_threshold=20):
     # Calculate overall pass/fail
     pass_fail_bool = (corr_final["Pass/Fail"] == "Pass").all()
 
-    return ({"Correlation Pair Table": corr_final}, fig, pass_fail_bool)
+    return (
+        {"Correlation Pair Table": corr_final},
+        fig,
+        pass_fail_bool,
+        RawData(
+            reference_correlations=corr_ref.to_dict(),
+            monitoring_correlations=corr_mon.to_dict(),
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
+    )

validmind/tests/ongoing_monitoring/PredictionProbabilitiesHistogramDrift.py

@@ -10,7 +10,7 @@ import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 from scipy import stats
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -201,4 +201,15 @@ def PredictionProbabilitiesHistogramDrift(
             }
         )
 
-    return fig, tables, all_passed
+    return (
+        fig,
+        tables,
+        all_passed,
+        RawData(
+            reference_probabilities=y_prob_ref,
+            monitoring_probabilities=y_prob_mon,
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
+    )

validmind/tests/ongoing_monitoring/ROCCurveDrift.py

@@ -8,7 +8,7 @@ import numpy as np
 import plotly.graph_objects as go
 from sklearn.metrics import roc_auc_score, roc_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -147,4 +147,18 @@ def ROCCurveDrift(datasets: List[VMDataset], model: VMModel):
         height=500,
     )
 
-    return fig1, fig2
+    return (
+        fig1,
+        fig2,
+        RawData(
+            fpr_ref=fpr_ref,
+            tpr_ref=tpr_ref,
+            auc_ref=auc_ref,
+            fpr_mon=fpr_mon,
+            tpr_mon=tpr_mon,
+            auc_mon=auc_mon,
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
+    )

validmind/tests/ongoing_monitoring/ScoreBandsDrift.py

@@ -7,7 +7,7 @@ from typing import List
 import numpy as np
 import pandas as pd
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -209,4 +209,13 @@ def ScoreBandsDrift(
         tables[table_name] = pd.DataFrame(rows)
         all_passed &= metric_passed
 
-    return tables, all_passed
+    # Collect raw data
+    raw_data = RawData(
+        ref_results=ref_results,
+        mon_results=mon_results,
+        model=model.input_id,
+        dataset_reference=datasets[0].input_id,
+        dataset_monitoring=datasets[1].input_id,
+    )
+
+    return tables, all_passed, raw_data

validmind/tests/ongoing_monitoring/TargetPredictionDistributionPlot.py

@@ -7,7 +7,7 @@ import plotly.figure_factory as ff
 import plotly.graph_objects as go
 from scipy.stats import kurtosis, skew
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization")
@@ -142,4 +142,15 @@ def TargetPredictionDistributionPlot(datasets, model, drift_pct_threshold=20):
 
     pass_fail_bool = (moments["Pass/Fail"] == "Pass").all()
 
-    return ({"Distribution Moments": moments}, fig, pass_fail_bool)
+    return (
+        {"Distribution Moments": moments},
+        fig,
+        pass_fail_bool,
+        RawData(
+            pred_ref=pred_ref,
+            pred_monitor=pred_monitor,
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
+    )

validmind/tests/prompt_validation/Clarity.py

@@ -106,5 +106,5 @@ def Clarity(model, min_threshold=7):
             }
         ],
         passed,
-        RawData(response=response),
+        RawData(response=response, model=model.input_id),
     )

validmind/tests/prompt_validation/NegativeInstruction.py

@@ -117,5 +117,5 @@ def NegativeInstruction(model, min_threshold=7):
             }
         ],
         passed,
-        RawData(model_response=response),
+        RawData(model_response=response, model=model.input_id),
     )

validmind/tests/prompt_validation/Robustness.py

@@ -130,5 +130,10 @@ def Robustness(model, dataset, num_tests=10):
     return (
         results,
         all(result["Pass/Fail"] == "Pass" for result in results),
-        RawData(generated_inputs=generated_inputs, responses=responses),
+        RawData(
+            generated_inputs=generated_inputs,
+            responses=responses,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/prompt_validation/Specificity.py

@@ -113,5 +113,5 @@ def Specificity(model, min_threshold=7):
             }
         ],
         passed,
-        RawData(response=response),
+        RawData(response=response, model=model.input_id),
     )

validmind/vm_models/result/utils.py

@@ -44,31 +44,12 @@ async def update_metadata(content_id: str, text: str, _json: Union[Dict, List] =
     content_id = parts[0]
     revision_name = parts[1] if len(parts) > 1 else None
 
-    # we always want composite metric definitions to be updated
-    should_update = content_id.startswith("composite_metric_def:")
-
-    # if we are updating a metric or test description, we check if the text
-    # has changed from the last time it was logged, and only update if it has
-    if content_id.split(":", 1)[0] in ["metric_description", "test_description"]:
-        try:
-            md = await api_client.aget_metadata(content_id)
-            # if there is an existing description, only update it if the new one
-            # is different and is an AI-generated description
-            should_update = (
-                md["text"] != text if revision_name == AI_REVISION_NAME else False
-            )
-            logger.debug(f"Check if description has changed: {should_update}")
-        except Exception:
-            # if exception, assume its not created yet TODO: don't catch all
-            should_update = True
-
-    if should_update:
-        if revision_name:
-            content_id = f"{content_id}::{revision_name}"
+    if revision_name:
+        content_id = f"{content_id}::{revision_name}"
 
-        logger.debug(f"Updating metadata for `{content_id}`")
+    logger.debug(f"Updating metadata for `{content_id}`")
 
-        await api_client.alog_metadata(content_id, text, _json)
+    await api_client.alog_metadata(content_id, text, _json)
 
 
 def check_for_sensitive_data(data: pd.DataFrame, inputs: List[VMInput]):

{validmind-2.8.10.dist-info → validmind-2.8.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: validmind
-Version: 2.8.10
+Version: 2.8.12
 Summary: ValidMind Library
 License: Commercial License
 Author: Andres Rodriguez
@@ -35,7 +35,7 @@ Requires-Dist: numba (<0.59.0)
 Requires-Dist: numpy
 Requires-Dist: openai (>=1)
 Requires-Dist: pandas (>=1.1,<=2.0.3)
-Requires-Dist: plotly
+Requires-Dist: plotly (<6.0.0)
 Requires-Dist: plotly-express
 Requires-Dist: polars
 Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"