validmind 2.7.5__py3-none-any.whl → 2.7.6__py3-none-any.whl

validmind/tests/ongoing_monitoring/PredictionCorrelation.py

@@ -2,16 +2,14 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-
-import matplotlib.pyplot as plt
-import numpy as np
-
+import pandas as pd
+import plotly.graph_objects as go
 from validmind import tags, tasks
 
 
 @tags("visualization")
 @tasks("monitoring")
-def PredictionCorrelation(datasets, model):
+def PredictionCorrelation(datasets, model, drift_pct_threshold=20):
     """
     Assesses correlation changes between model predictions from reference and monitoring datasets to detect potential
     target drift.
@@ -47,55 +45,98 @@ def PredictionCorrelation(datasets, model):
     - Focuses solely on linear relationships, potentially missing non-linear interactions.
     """
 
-    prediction_prob_column = f"{model.input_id}_probabilities"
-    prediction_column = f"{model.input_id}_prediction"
+    # Get feature columns and predictions
+    feature_columns = datasets[0].feature_columns
+    y_prob_ref = pd.Series(datasets[0].y_prob(model), index=datasets[0].df.index)
+    y_prob_mon = pd.Series(datasets[1].y_prob(model), index=datasets[1].df.index)
 
-    df_corr = datasets[0]._df.corr()
-    df_corr = df_corr[[prediction_prob_column]]
+    # Create dataframes with features and predictions
+    df_ref = datasets[0].df[feature_columns].copy()
+    df_ref["predictions"] = y_prob_ref
 
-    df_corr2 = datasets[1]._df.corr()
-    df_corr2 = df_corr2[[prediction_prob_column]]
+    df_mon = datasets[1].df[feature_columns].copy()
+    df_mon["predictions"] = y_prob_mon
 
-    corr_final = df_corr.merge(df_corr2, left_index=True, right_index=True)
-    corr_final.columns = ["Reference Predictions", "Monitoring Predictions"]
-    corr_final = corr_final.drop(index=[prediction_column, prediction_prob_column])
+    # Calculate correlations
+    corr_ref = df_ref.corr()["predictions"]
+    corr_mon = df_mon.corr()["predictions"]
 
-    n = len(corr_final)
-    r = np.arange(n)
-    width = 0.25
+    # Combine correlations (excluding the predictions row)
+    corr_final = pd.DataFrame(
+        {
+            "Reference Predictions": corr_ref[feature_columns],
+            "Monitoring Predictions": corr_mon[feature_columns],
+        }
+    )
 
-    fig = plt.figure()
+    # Calculate drift percentage with direction
+    corr_final["Drift (%)"] = (
+        (corr_final["Monitoring Predictions"] - corr_final["Reference Predictions"])
+        / corr_final["Reference Predictions"].abs()
+        * 100
+    ).round(2)
+
+    # Add Pass/Fail column based on absolute drift
+    corr_final["Pass/Fail"] = (
+        corr_final["Drift (%)"]
+        .abs()
+        .apply(lambda x: "Pass" if x < drift_pct_threshold else "Fail")
+    )
 
-    plt.bar(
-        r,
-        corr_final["Reference Predictions"],
-        color="b",
-        width=width,
-        edgecolor="black",
-        label="Reference Prediction Correlation",
+    # Create plotly figure
+    fig = go.Figure()
+
+    # Add reference predictions bar
+    fig.add_trace(
+        go.Bar(
+            name="Reference Prediction Correlation",
+            x=corr_final.index,
+            y=corr_final["Reference Predictions"],
+            marker_color="blue",
+            marker_line_color="black",
+            marker_line_width=1,
+            opacity=0.75,
+        )
     )
-    plt.bar(
-        r + width,
-        corr_final["Monitoring Predictions"],
-        color="g",
-        width=width,
-        edgecolor="black",
-        label="Monitoring Prediction Correlation",
+
+    # Add monitoring predictions bar
+    fig.add_trace(
+        go.Bar(
+            name="Monitoring Prediction Correlation",
+            x=corr_final.index,
+            y=corr_final["Monitoring Predictions"],
+            marker_color="green",
+            marker_line_color="black",
+            marker_line_width=1,
+            opacity=0.75,
+        )
     )
 
-    plt.xlabel("Features")
-    plt.ylabel("Correlation")
-    plt.title("Correlation between Predictions and Features")
+    # Update layout
+    fig.update_layout(
+        title="Correlation between Predictions and Features",
+        xaxis_title="Features",
+        yaxis_title="Correlation",
+        barmode="group",
+        template="plotly_white",
+        showlegend=True,
+        xaxis_tickangle=-45,
+        yaxis=dict(
+            range=[-1, 1],  # Correlation range is always -1 to 1
+            zeroline=True,
+            zerolinewidth=1,
+            zerolinecolor="grey",
+            gridcolor="lightgrey",
+        ),
+        hoverlabel=dict(bgcolor="white", font_size=12, font_family="Arial"),
+    )
 
-    features = corr_final.index.to_list()
-    plt.xticks(r + width / 2, features, rotation=45)
-    plt.legend()
-    plt.tight_layout()
+    # Ensure Features is the first column
+    corr_final["Feature"] = corr_final.index
+    cols = ["Feature"] + [col for col in corr_final.columns if col != "Feature"]
+    corr_final = corr_final[cols]
 
-    plt.close()
+    # Calculate overall pass/fail
+    pass_fail_bool = (corr_final["Pass/Fail"] == "Pass").all()
 
-    corr_final["Features"] = corr_final.index
-    corr_final = corr_final[
-        ["Features", "Reference Predictions", "Monitoring Predictions"]
-    ]
-    return ({"Correlation Pair Table": corr_final}, fig)
+    return ({"Correlation Pair Table": corr_final}, fig, pass_fail_bool)

validmind/tests/ongoing_monitoring/PredictionProbabilitiesHistogramDrift.py

@@ -0,0 +1,202 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+from scipy import stats
+from typing import List
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tags("visualization", "credit_risk")
+@tasks("classification")
+def PredictionProbabilitiesHistogramDrift(
+    datasets: List[VMDataset],
+    model: VMModel,
+    title="Prediction Probabilities Histogram Drift",
+    drift_pct_threshold: float = 20.0,
+):
+    """
+    Compares prediction probability distributions between reference and monitoring datasets.
+
+    ### Purpose
+
+    The Prediction Probabilities Histogram Drift test is designed to evaluate changes in the model's
+    probability predictions over time. By comparing probability distributions between reference and
+    monitoring datasets using histograms, this test helps identify whether the model's probability
+    assignments have shifted in production. This is crucial for understanding if the model's risk
+    assessment behavior remains consistent and whether its probability estimates maintain their
+    original distribution patterns.
+
+    ### Test Mechanism
+
+    This test proceeds by generating histograms of prediction probabilities for both reference and
+    monitoring datasets. For each class, it analyzes the distribution shape, central tendency, and
+    spread of probabilities. The test computes distribution moments (mean, variance, skewness,
+    kurtosis) and quantifies their drift between datasets. Visual comparison of overlaid histograms
+    provides immediate insight into distribution changes.
+
+    ### Signs of High Risk
+
+    - Significant shifts in probability distribution shapes
+    - Large drifts in distribution moments exceeding threshold
+    - Appearance of new modes or peaks in monitoring data
+    - Changes in the spread or concentration of probabilities
+    - Systematic shifts in probability assignments
+    - Unexpected changes in distribution characteristics
+
+    ### Strengths
+
+    - Provides intuitive visualization of probability changes
+    - Identifies specific changes in distribution shape
+    - Enables quantitative assessment of distribution drift
+    - Supports analysis across multiple classes
+    - Includes comprehensive moment analysis
+    - Maintains interpretable probability scale
+
+    ### Limitations
+
+    - May be sensitive to binning choices
+    - Requires sufficient samples for reliable histograms
+    - Cannot suggest probability recalibration
+    - Complex interpretation for multiple classes
+    - May not capture subtle distribution changes
+    - Limited to univariate probability analysis
+    """
+    # Get predictions and true values
+    y_prob_ref = datasets[0].y_prob(model)
+    df_ref = datasets[0].df.copy()
+    df_ref["probabilities"] = y_prob_ref
+
+    y_prob_mon = datasets[1].y_prob(model)
+    df_mon = datasets[1].df.copy()
+    df_mon["probabilities"] = y_prob_mon
+
+    # Get unique classes
+    classes = sorted(df_ref[datasets[0].target_column].unique())
+
+    # Create subplots with more horizontal space for legends
+    fig = make_subplots(
+        rows=len(classes),
+        cols=1,
+        subplot_titles=[f"Class {cls}" for cls in classes],
+        horizontal_spacing=0.15,
+    )
+
+    # Define colors
+    ref_color = "rgba(31, 119, 180, 0.8)"  # Blue with 0.8 opacity
+    mon_color = "rgba(255, 127, 14, 0.8)"  # Orange with 0.8 opacity
+
+    # Dictionary to store tables for each class
+    tables = {}
+    all_passed = True  # Track overall pass/fail
+
+    # Add histograms and create tables for each class
+    for i, class_value in enumerate(classes, start=1):
+        # Get probabilities for current class
+        ref_probs = df_ref[df_ref[datasets[0].target_column] == class_value][
+            "probabilities"
+        ]
+        mon_probs = df_mon[df_mon[datasets[1].target_column] == class_value][
+            "probabilities"
+        ]
+
+        # Calculate distribution moments
+        ref_stats = {
+            "Mean": np.mean(ref_probs),
+            "Variance": np.var(ref_probs),
+            "Skewness": stats.skew(ref_probs),
+            "Kurtosis": stats.kurtosis(ref_probs),
+        }
+
+        mon_stats = {
+            "Mean": np.mean(mon_probs),
+            "Variance": np.var(mon_probs),
+            "Skewness": stats.skew(mon_probs),
+            "Kurtosis": stats.kurtosis(mon_probs),
+        }
+
+        # Create table for this class
+        table_data = []
+        class_passed = True  # Track pass/fail for this class
+
+        for stat_name in ["Mean", "Variance", "Skewness", "Kurtosis"]:
+            ref_val = ref_stats[stat_name]
+            mon_val = mon_stats[stat_name]
+            drift = (
+                ((mon_val - ref_val) / abs(ref_val)) * 100 if ref_val != 0 else np.inf
+            )
+            passed = abs(drift) < drift_pct_threshold
+            class_passed &= passed  # Update class pass/fail
+
+            table_data.append(
+                {
+                    "Statistic": stat_name,
+                    "Reference": round(ref_val, 4),
+                    "Monitoring": round(mon_val, 4),
+                    "Drift (%)": round(drift, 2),
+                    "Pass/Fail": "Pass" if passed else "Fail",
+                }
+            )
+
+        tables[f"Class {class_value}"] = pd.DataFrame(table_data)
+        all_passed &= class_passed  # Update overall pass/fail
+
+        # Reference dataset histogram
+        fig.add_trace(
+            go.Histogram(
+                x=ref_probs,
+                name=f"Reference - Class {class_value}",
+                marker_color=ref_color,
+                showlegend=True,
+                legendrank=i * 2 - 1,
+            ),
+            row=i,
+            col=1,
+        )
+
+        # Monitoring dataset histogram
+        fig.add_trace(
+            go.Histogram(
+                x=mon_probs,
+                name=f"Monitoring - Class {class_value}",
+                marker_color=mon_color,
+                showlegend=True,
+                legendrank=i * 2,
+            ),
+            row=i,
+            col=1,
+        )
+
+    # Update layout
+    fig.update_layout(
+        title_text=title,
+        barmode="overlay",
+        height=300 * len(classes),
+        width=1000,
+        showlegend=True,
+    )
+
+    # Update axes labels and add separate legends for each subplot
+    for i in range(len(classes)):
+        fig.update_xaxes(title_text="Probability", row=i + 1, col=1)
+        fig.update_yaxes(title_text="Frequency", row=i + 1, col=1)
+
+        # Add separate legend for each subplot
+        fig.update_layout(
+            **{
+                f'legend{i+1 if i > 0 else ""}': dict(
+                    yanchor="middle",
+                    y=1 - (i / len(classes)) - (0.5 / len(classes)),
+                    xanchor="left",
+                    x=1.05,
+                    tracegroupgap=5,
+                )
+            }
+        )
+
+    return fig, tables, all_passed

validmind/tests/ongoing_monitoring/PredictionQuantilesAcrossFeatures.py

@@ -0,0 +1,97 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+from validmind import tags, tasks
+
+
+@tags("visualization")
+@tasks("monitoring")
+def PredictionQuantilesAcrossFeatures(datasets, model):
+    """
+    Assesses differences in model prediction distributions across individual features between reference
+    and monitoring datasets through quantile analysis.
+
+    ### Purpose
+
+    This test aims to visualize how prediction distributions vary across feature values by showing
+    quantile information between reference and monitoring datasets. It helps identify significant
+    shifts in prediction patterns and potential areas of model instability.
+
+    ### Test Mechanism
+
+    The test generates box plots for each feature, comparing prediction probability distributions
+    between the reference and monitoring datasets. Each plot consists of two subplots showing the
+    quantile distribution of predictions: one for reference data and one for monitoring data.
+
+    ### Signs of High Risk
+
+    - Significant differences in prediction distributions between reference and monitoring data
+    - Unexpected shifts in prediction quantiles across feature values
+    - Large changes in prediction variability between datasets
+
+    ### Strengths
+
+    - Provides clear visualization of prediction distribution changes
+    - Shows outliers and variability in predictions across features
+    - Enables quick identification of problematic feature ranges
+
+    ### Limitations
+
+    - May not capture complex relationships between features and predictions
+    - Quantile analysis may smooth over important individual predictions
+    - Requires careful interpretation of distribution changes
+    """
+
+    feature_columns = datasets[0].feature_columns
+    y_prob_reference = datasets[0].y_prob(model)
+    y_prob_monitoring = datasets[1].y_prob(model)
+
+    figures_to_save = []
+    for column in feature_columns:
+        # Create subplot
+        fig = make_subplots(1, 2, subplot_titles=("Reference", "Monitoring"))
+
+        # Add reference box plot
+        fig.add_trace(
+            go.Box(
+                x=datasets[0].df[column],
+                y=y_prob_reference,
+                name="Reference",
+                boxpoints="outliers",
+                marker_color="blue",
+            ),
+            row=1,
+            col=1,
+        )
+
+        # Add monitoring box plot
+        fig.add_trace(
+            go.Box(
+                x=datasets[1].df[column],
+                y=y_prob_monitoring,
+                name="Monitoring",
+                boxpoints="outliers",
+                marker_color="red",
+            ),
+            row=1,
+            col=2,
+        )
+
+        # Update layout
+        fig.update_layout(
+            title=f"Prediction Distributions vs {column}",
+            showlegend=False,
+            width=800,
+            height=400,
+        )
+
+        # Update axes
+        fig.update_xaxes(title=column)
+        fig.update_yaxes(title="Prediction Value")
+
+        figures_to_save.append(fig)
+
+    return tuple(figures_to_save)

validmind/tests/ongoing_monitoring/ROCCurveDrift.py

@@ -0,0 +1,149 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import plotly.graph_objects as go
+from sklearn.metrics import roc_auc_score, roc_curve
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.vm_models import VMDataset, VMModel
+
+from typing import List
+
+
+@tags(
+    "sklearn",
+    "binary_classification",
+    "model_performance",
+    "visualization",
+)
+@tasks("classification", "text_classification")
+def ROCCurveDrift(datasets: List[VMDataset], model: VMModel):
+    """
+    Compares ROC curves between reference and monitoring datasets.
+
+    ### Purpose
+
+    The ROC Curve Drift test is designed to evaluate changes in the model's discriminative ability
+    over time. By comparing Receiver Operating Characteristic (ROC) curves between reference and
+    monitoring datasets, this test helps identify whether the model maintains its ability to
+    distinguish between classes across different decision thresholds. This is crucial for
+    understanding if the model's trade-off between sensitivity and specificity remains stable
+    in production.
+
+    ### Test Mechanism
+
+    This test proceeds by generating ROC curves for both reference and monitoring datasets. For each
+    dataset, it plots the True Positive Rate against the False Positive Rate across all possible
+    classification thresholds. The test also computes AUC scores and visualizes the difference
+    between ROC curves, providing both graphical and numerical assessments of discrimination
+    stability. Special attention is paid to regions where curves diverge significantly.
+
+    ### Signs of High Risk
+
+    - Large differences between reference and monitoring ROC curves
+    - Significant drop in AUC score for monitoring dataset
+    - Systematic differences in specific FPR regions
+    - Changes in optimal operating points
+    - Inconsistent performance across different thresholds
+    - Unexpected crossovers between curves
+
+    ### Strengths
+
+    - Provides comprehensive view of discriminative ability
+    - Identifies specific threshold ranges with drift
+    - Enables visualization of performance differences
+    - Includes AUC comparison for overall assessment
+    - Supports threshold-independent evaluation
+    - Maintains interpretable performance metrics
+
+    ### Limitations
+
+    - Limited to binary classification problems
+    - May be sensitive to class distribution changes
+    - Cannot suggest optimal threshold adjustments
+    - Requires visual inspection for detailed analysis
+    - Complex interpretation of curve differences
+    - May not capture subtle performance changes
+    """
+    # Check for binary classification
+    if len(np.unique(datasets[0].y)) > 2:
+        raise SkipTestError(
+            "ROC Curve Drift is only supported for binary classification models"
+        )
+
+    # Calculate ROC curves for reference dataset
+    y_prob_ref = datasets[0].y_prob(model)
+    y_true_ref = datasets[0].y.astype(y_prob_ref.dtype).flatten()
+    fpr_ref, tpr_ref, _ = roc_curve(y_true_ref, y_prob_ref, drop_intermediate=False)
+    auc_ref = roc_auc_score(y_true_ref, y_prob_ref)
+
+    # Calculate ROC curves for monitoring dataset
+    y_prob_mon = datasets[1].y_prob(model)
+    y_true_mon = datasets[1].y.astype(y_prob_mon.dtype).flatten()
+    fpr_mon, tpr_mon, _ = roc_curve(y_true_mon, y_prob_mon, drop_intermediate=False)
+    auc_mon = roc_auc_score(y_true_mon, y_prob_mon)
+
+    # Create superimposed ROC curves plot
+    fig1 = go.Figure()
+
+    fig1.add_trace(
+        go.Scatter(
+            x=fpr_ref,
+            y=tpr_ref,
+            mode="lines",
+            name=f"Reference (AUC = {auc_ref:.3f})",
+            line=dict(color="blue", width=2),
+        )
+    )
+
+    fig1.add_trace(
+        go.Scatter(
+            x=fpr_mon,
+            y=tpr_mon,
+            mode="lines",
+            name=f"Monitoring (AUC = {auc_mon:.3f})",
+            line=dict(color="red", width=2),
+        )
+    )
+
+    fig1.update_layout(
+        title="ROC Curves Comparison",
+        xaxis=dict(title="False Positive Rate"),
+        yaxis=dict(title="True Positive Rate"),
+        width=700,
+        height=500,
+    )
+
+    # Interpolate monitoring TPR to match reference FPR points
+    tpr_mon_interp = np.interp(fpr_ref, fpr_mon, tpr_mon)
+
+    # Calculate TPR difference
+    tpr_diff = tpr_mon_interp - tpr_ref
+
+    # Create difference plot
+    fig2 = go.Figure()
+
+    fig2.add_trace(
+        go.Scatter(
+            x=fpr_ref,
+            y=tpr_diff,
+            mode="lines",
+            name="TPR Difference",
+            line=dict(color="purple", width=2),
+        )
+    )
+
+    # Add horizontal line at y=0
+    fig2.add_hline(y=0, line=dict(color="grey", dash="dash"), name="No Difference")
+
+    fig2.update_layout(
+        title="ROC Curve Difference (Monitoring - Reference)",
+        xaxis=dict(title="False Positive Rate"),
+        yaxis=dict(title="TPR Difference"),
+        width=700,
+        height=500,
+    )
+
+    return fig1, fig2