validmind 2.7.5__py3-none-any.whl → 2.7.6__py3-none-any.whl

validmind/tests/ongoing_monitoring/ConfusionMatrixDrift.py

@@ -0,0 +1,191 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import confusion_matrix
+from typing import List
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tags(
+    "sklearn", "binary_classification", "multiclass_classification", "model_performance"
+)
+@tasks("classification", "text_classification")
+def ConfusionMatrixDrift(
+    datasets: List[VMDataset], model: VMModel, drift_pct_threshold=20
+):
+    """
+    Compares confusion matrix metrics between reference and monitoring datasets.
+
+    ### Purpose
+
+    The Confusion Matrix Drift test is designed to evaluate changes in the model's error patterns
+    over time. By comparing confusion matrix elements between reference and monitoring datasets, this
+    test helps identify whether the model maintains consistent prediction behavior in production. This
+    is crucial for understanding if the model's error patterns have shifted and whether specific types
+    of misclassifications have become more prevalent.
+
+    ### Test Mechanism
+
+    This test proceeds by generating confusion matrices for both reference and monitoring datasets.
+    For binary classification, it tracks True Positives, True Negatives, False Positives, and False
+    Negatives as percentages of total predictions. For multiclass problems, it analyzes per-class
+    metrics including true positives and error rates. The test quantifies drift as percentage changes
+    in these metrics between datasets, providing detailed insight into shifting prediction patterns.
+
+    ### Signs of High Risk
+
+    - Large drifts in confusion matrix elements exceeding threshold
+    - Systematic changes in false positive or false negative rates
+    - Inconsistent changes across different classes
+    - Significant shifts in error patterns for specific classes
+    - Unexpected improvements in certain metrics
+    - Divergent trends between different types of errors
+
+    ### Strengths
+
+    - Provides detailed analysis of prediction behavior
+    - Identifies specific types of prediction changes
+    - Enables early detection of systematic errors
+    - Includes comprehensive error pattern analysis
+    - Supports both binary and multiclass problems
+    - Maintains interpretable percentage-based metrics
+
+    ### Limitations
+
+    - May be sensitive to class distribution changes
+    - Cannot identify root causes of prediction drift
+    - Requires sufficient samples for reliable comparison
+    - Limited to hard predictions (not probabilities)
+    - May not capture subtle changes in decision boundaries
+    - Complex interpretation for multiclass problems
+    """
+    # Get predictions and true values for reference dataset
+    y_pred_ref = datasets[0].y_pred(model)
+    y_true_ref = datasets[0].y.astype(y_pred_ref.dtype)
+
+    # Get predictions and true values for monitoring dataset
+    y_pred_mon = datasets[1].y_pred(model)
+    y_true_mon = datasets[1].y.astype(y_pred_mon.dtype)
+
+    # Get unique labels from reference dataset
+    labels = np.unique(y_true_ref)
+    labels = sorted(labels.tolist())
+
+    # Calculate confusion matrices
+    cm_ref = confusion_matrix(y_true_ref, y_pred_ref, labels=labels)
+    cm_mon = confusion_matrix(y_true_mon, y_pred_mon, labels=labels)
+
+    # Get total counts
+    total_ref = len(y_true_ref)
+    total_mon = len(y_true_mon)
+
+    # Create sample counts table
+    counts_data = {
+        "Dataset": ["Reference", "Monitoring"],
+        "Total": [total_ref, total_mon],
+    }
+
+    # Add per-class counts
+    for label in labels:
+        label_str = f"Class_{label}"
+        counts_data[label_str] = [
+            np.sum(y_true_ref == label),
+            np.sum(y_true_mon == label),
+        ]
+
+    counts_df = pd.DataFrame(counts_data)
+
+    # Create confusion matrix metrics
+    metrics = []
+
+    if len(labels) == 2:
+        # Binary classification
+        tn_ref, fp_ref, fn_ref, tp_ref = cm_ref.ravel()
+        tn_mon, fp_mon, fn_mon, tp_mon = cm_mon.ravel()
+
+        confusion_elements = [
+            ("True Negatives (%)", tn_ref / total_ref * 100, tn_mon / total_mon * 100),
+            ("False Positives (%)", fp_ref / total_ref * 100, fp_mon / total_mon * 100),
+            ("False Negatives (%)", fn_ref / total_ref * 100, fn_mon / total_mon * 100),
+            ("True Positives (%)", tp_ref / total_ref * 100, tp_mon / total_mon * 100),
+        ]
+
+        for name, ref_val, mon_val in confusion_elements:
+            metrics.append(
+                {
+                    "Metric": name,
+                    "Reference": round(ref_val, 2),
+                    "Monitoring": round(mon_val, 2),
+                }
+            )
+
+    else:
+        # Multiclass - calculate per-class metrics
+        for i, label in enumerate(labels):
+            # True Positives for this class
+            tp_ref = cm_ref[i, i]
+            tp_mon = cm_mon[i, i]
+
+            # False Positives (sum of column minus TP)
+            fp_ref = cm_ref[:, i].sum() - tp_ref
+            fp_mon = cm_mon[:, i].sum() - tp_mon
+
+            # False Negatives (sum of row minus TP)
+            fn_ref = cm_ref[i, :].sum() - tp_ref
+            fn_mon = cm_mon[i, :].sum() - tp_mon
+
+            class_metrics = [
+                (
+                    f"True Positives_{label} (%)",
+                    tp_ref / total_ref * 100,
+                    tp_mon / total_mon * 100,
+                ),
+                (
+                    f"False Positives_{label} (%)",
+                    fp_ref / total_ref * 100,
+                    fp_mon / total_mon * 100,
+                ),
+                (
+                    f"False Negatives_{label} (%)",
+                    fn_ref / total_ref * 100,
+                    fn_mon / total_mon * 100,
+                ),
+            ]
+
+            for name, ref_val, mon_val in class_metrics:
+                metrics.append(
+                    {
+                        "Metric": name,
+                        "Reference": round(ref_val, 2),
+                        "Monitoring": round(mon_val, 2),
+                    }
+                )
+
+    # Create metrics DataFrame
+    metrics_df = pd.DataFrame(metrics)
+
+    # Calculate drift percentage with direction
+    metrics_df["Drift (%)"] = (
+        (metrics_df["Monitoring"] - metrics_df["Reference"])
+        / metrics_df["Reference"].abs()
+        * 100
+    ).round(2)
+
+    # Add Pass/Fail column based on absolute drift
+    metrics_df["Pass/Fail"] = (
+        metrics_df["Drift (%)"]
+        .abs()
+        .apply(lambda x: "Pass" if x < drift_pct_threshold else "Fail")
+    )
+
+    # Calculate overall pass/fail
+    pass_fail_bool = (metrics_df["Pass/Fail"] == "Pass").all()
+
+    return (
+        {"Confusion Matrix Metrics": metrics_df, "Sample Counts": counts_df},
+        pass_fail_bool,
+    )

validmind/tests/ongoing_monitoring/CumulativePredictionProbabilitiesDrift.py

@@ -0,0 +1,176 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+from typing import List
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tags("visualization", "credit_risk")
+@tasks("classification")
+def CumulativePredictionProbabilitiesDrift(
+    datasets: List[VMDataset],
+    model: VMModel,
+):
+    """
+    Compares cumulative prediction probability distributions between reference and monitoring datasets.
+
+    ### Purpose
+
+    The Cumulative Prediction Probabilities Drift test is designed to evaluate changes in the model's
+    probability predictions over time. By comparing cumulative distribution functions of predicted
+    probabilities between reference and monitoring datasets, this test helps identify whether the
+    model's probability assignments remain stable in production. This is crucial for understanding if
+    the model's risk assessment behavior has shifted and whether its probability calibration remains
+    consistent.
+
+    ### Test Mechanism
+
+    This test proceeds by generating cumulative distribution functions (CDFs) of predicted probabilities
+    for both reference and monitoring datasets. For each class, it plots the cumulative proportion of
+    predictions against probability values, enabling direct comparison of probability distributions.
+    The test visualizes both the CDFs and their differences, providing insight into how probability
+    assignments have shifted across the entire probability range.
+
+    ### Signs of High Risk
+
+    - Large gaps between reference and monitoring CDFs
+    - Systematic shifts in probability assignments
+    - Concentration of differences in specific probability ranges
+    - Changes in the shape of probability distributions
+    - Unexpected patterns in cumulative differences
+    - Significant shifts in probability thresholds
+
+    ### Strengths
+
+    - Provides comprehensive view of probability changes
+    - Identifies specific probability ranges with drift
+    - Enables visualization of distribution differences
+    - Supports analysis across multiple classes
+    - Maintains interpretable probability scale
+    - Captures subtle changes in probability assignments
+
+    ### Limitations
+
+    - Does not provide single drift metric
+    - May be complex to interpret for multiple classes
+    - Cannot suggest probability recalibration
+    - Requires visual inspection for assessment
+    - Sensitive to sample size differences
+    - May not capture class-specific calibration issues
+    """
+    # Get predictions and true values
+    y_prob_ref = datasets[0].y_prob(model)
+    df_ref = datasets[0].df.copy()
+    df_ref["probabilities"] = y_prob_ref
+
+    y_prob_mon = datasets[1].y_prob(model)
+    df_mon = datasets[1].df.copy()
+    df_mon["probabilities"] = y_prob_mon
+
+    # Get unique classes
+    classes = sorted(df_ref[datasets[0].target_column].unique())
+
+    # Define colors
+    ref_color = "rgba(31, 119, 180, 0.8)"  # Blue with 0.8 opacity
+    mon_color = "rgba(255, 127, 14, 0.8)"  # Orange with 0.8 opacity
+    diff_color = "rgba(148, 103, 189, 0.8)"  # Purple with 0.8 opacity
+
+    figures = []
+    for class_value in classes:
+        # Create figure with secondary y-axis
+        fig = make_subplots(
+            rows=2,
+            cols=1,
+            subplot_titles=[
+                f"Cumulative Distributions - Class {class_value}",
+                "Difference (Monitoring - Reference)",
+            ],
+            vertical_spacing=0.15,
+            shared_xaxes=True,
+        )
+
+        # Get probabilities for current class
+        ref_probs = df_ref[df_ref[datasets[0].target_column] == class_value][
+            "probabilities"
+        ]
+        mon_probs = df_mon[df_mon[datasets[1].target_column] == class_value][
+            "probabilities"
+        ]
+
+        # Calculate cumulative distributions
+        ref_sorted = np.sort(ref_probs)
+        ref_cumsum = np.arange(len(ref_sorted)) / float(len(ref_sorted))
+
+        mon_sorted = np.sort(mon_probs)
+        mon_cumsum = np.arange(len(mon_sorted)) / float(len(mon_sorted))
+
+        # Reference dataset cumulative curve
+        fig.add_trace(
+            go.Scatter(
+                x=ref_sorted,
+                y=ref_cumsum,
+                mode="lines",
+                name="Reference",
+                line=dict(color=ref_color, width=2),
+            ),
+            row=1,
+            col=1,
+        )
+
+        # Monitoring dataset cumulative curve
+        fig.add_trace(
+            go.Scatter(
+                x=mon_sorted,
+                y=mon_cumsum,
+                mode="lines",
+                name="Monitoring",
+                line=dict(color=mon_color, width=2),
+            ),
+            row=1,
+            col=1,
+        )
+
+        # Calculate and plot difference
+        # Interpolate monitoring values to match reference x-points
+        mon_interp = np.interp(ref_sorted, mon_sorted, mon_cumsum)
+        difference = mon_interp - ref_cumsum
+
+        fig.add_trace(
+            go.Scatter(
+                x=ref_sorted,
+                y=difference,
+                mode="lines",
+                name="Difference",
+                line=dict(color=diff_color, width=2),
+            ),
+            row=2,
+            col=1,
+        )
+
+        # Add horizontal line at y=0 for difference plot
+        fig.add_hline(y=0, line=dict(color="grey", dash="dash"), row=2, col=1)
+
+        # Update layout
+        fig.update_layout(
+            height=600,
+            width=800,
+            showlegend=True,
+            legend=dict(yanchor="middle", y=0.9, xanchor="left", x=1.05),
+        )
+
+        # Update axes
+        fig.update_xaxes(title_text="Probability", range=[0, 1], row=2, col=1)
+        fig.update_xaxes(range=[0, 1], row=1, col=1)
+        fig.update_yaxes(
+            title_text="Cumulative Distribution", range=[0, 1], row=1, col=1
+        )
+        fig.update_yaxes(title_text="Difference", row=2, col=1)
+
+        figures.append(fig)
+
+    return tuple(figures)

validmind/tests/ongoing_monitoring/FeatureDrift.py

@@ -2,18 +2,99 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-
-import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-
+import plotly.graph_objects as go
 from validmind import tags, tasks
 
 
+def calculate_psi_score(actual, expected):
+    """Calculate PSI score for a single bucket."""
+    return (actual - expected) * np.log((actual + 1e-6) / (expected + 1e-6))
+
+
+def calculate_feature_distributions(
+    reference_data, monitoring_data, feature_columns, bins
+):
+    """Calculate population distributions for each feature."""
+    # Calculate quantiles from reference data
+    quantiles = reference_data[feature_columns].quantile(
+        bins, method="single", interpolation="nearest"
+    )
+
+    distributions = {}
+    for dataset_name, data in [
+        ("reference", reference_data),
+        ("monitoring", monitoring_data),
+    ]:
+        for feature in feature_columns:
+            for bin_idx, threshold in enumerate(quantiles[feature]):
+                if bin_idx == 0:
+                    mask = data[feature] < threshold
+                else:
+                    prev_threshold = quantiles[feature][bins[bin_idx - 1]]
+                    mask = (data[feature] >= prev_threshold) & (
+                        data[feature] < threshold
+                    )
+
+                count = mask.sum()
+                proportion = count / len(data)
+                distributions[(dataset_name, feature, bins[bin_idx])] = proportion
+
+    return distributions
+
+
+def create_distribution_plot(feature_name, reference_dist, monitoring_dist, bins):
+    """Create population distribution plot for a feature."""
+    fig = go.Figure()
+
+    # Add reference distribution
+    fig.add_trace(
+        go.Bar(
+            x=list(range(len(bins))),
+            y=reference_dist,
+            name="Reference",
+            marker_color="blue",
+            marker_line_color="black",
+            marker_line_width=1,
+            opacity=0.75,
+        )
+    )
+
+    # Add monitoring distribution
+    fig.add_trace(
+        go.Bar(
+            x=list(range(len(bins))),
+            y=monitoring_dist,
+            name="Monitoring",
+            marker_color="green",
+            marker_line_color="black",
+            marker_line_width=1,
+            opacity=0.75,
+        )
+    )
+
+    fig.update_layout(
+        title=f"Population Distribution: {feature_name}",
+        xaxis_title="Bin",
+        yaxis_title="Population %",
+        barmode="group",
+        template="plotly_white",
+        showlegend=True,
+        width=800,
+        height=400,
+    )
+
+    return fig
+
+
 @tags("visualization")
 @tasks("monitoring")
 def FeatureDrift(
-    datasets, bins=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], feature_columns=None
+    datasets,
+    bins=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
+    feature_columns=None,
+    psi_threshold=0.2,
 ):
     """
     Evaluates changes in feature distribution over time to identify potential model drift.
@@ -57,130 +138,48 @@ def FeatureDrift(
     - PSI score interpretation can be overly simplistic for complex datasets.
     """
 
-    # Feature columns for both datasets should be the same if not given
-    default_feature_columns = datasets[0].feature_columns
-    feature_columns = feature_columns or default_feature_columns
+    # Get feature columns
+    feature_columns = feature_columns or datasets[0].feature_columns
 
-    x_train_df = datasets[0].x_df()
-    x_test_df = datasets[1].x_df()
+    # Get data
+    reference_data = datasets[0].df
+    monitoring_data = datasets[1].df
 
-    quantiles_train = x_train_df[feature_columns].quantile(
-        bins, method="single", interpolation="nearest"
-    )
-    PSI_QUANTILES = quantiles_train.to_dict()
-
-    PSI_BUCKET_FRAC, col, n = get_psi_buckets(
-        x_test_df, x_train_df, feature_columns, bins, PSI_QUANTILES
+    # Calculate distributions
+    distributions = calculate_feature_distributions(
+        reference_data, monitoring_data, feature_columns, bins
     )
 
-    def nest(d: dict) -> dict:
-        result = {}
-        for key, value in d.items():
-            target = result
-            for k in key[:-1]:  # traverse all keys but the last
-                target = target.setdefault(k, {})
-            target[key[-1]] = value
-        return result
-
-    PSI_BUCKET_FRAC = nest(PSI_BUCKET_FRAC)
-
-    PSI_SCORES = {}
-    for col in feature_columns:
+    # Calculate PSI scores
+    psi_scores = {}
+    for feature in feature_columns:
         psi = 0
-        for n in bins:
-            actual = PSI_BUCKET_FRAC["test"][col][n]
-            expected = PSI_BUCKET_FRAC["train"][col][n]
-            psi_of_bucket = (actual - expected) * np.log(
-                (actual + 1e-6) / (expected + 1e-6)
-            )
-            psi += psi_of_bucket
-        PSI_SCORES[col] = psi
-
-    psi_df = pd.DataFrame(list(PSI_SCORES.items()), columns=["Features", "PSI Score"])
+        for bin_val in bins:
+            reference_prop = distributions[("reference", feature, bin_val)]
+            monitoring_prop = distributions[("monitoring", feature, bin_val)]
+            psi += calculate_psi_score(monitoring_prop, reference_prop)
+        psi_scores[feature] = psi
+
+    # Create PSI score dataframe
+    psi_df = pd.DataFrame(list(psi_scores.items()), columns=["Feature", "PSI Score"])
+
+    # Add Pass/Fail column
+    psi_df["Pass/Fail"] = psi_df["PSI Score"].apply(
+        lambda x: "Pass" if x < psi_threshold else "Fail"
+    )
 
+    # Sort by PSI Score
     psi_df.sort_values(by=["PSI Score"], inplace=True, ascending=False)
 
-    psi_table = [
-        {"Features": values["Features"], "PSI Score": values["PSI Score"]}
-        for i, values in enumerate(psi_df.to_dict(orient="records"))
-    ]
-
-    save_fig = plot_hist(PSI_BUCKET_FRAC, bins)
-
-    final_psi = pd.DataFrame(psi_table)
-
-    return (final_psi, *save_fig)
-
-
-def get_psi_buckets(x_test_df, x_train_df, feature_columns, bins, PSI_QUANTILES):
-    DATA = {"test": x_test_df, "train": x_train_df}
-    PSI_BUCKET_FRAC = {}
-    for table in DATA.keys():
-        total_count = DATA[table].shape[0]
-        for col in feature_columns:
-            count_sum = 0
-            for n in bins:
-                if n == 0:
-                    bucket_count = (DATA[table][col] < PSI_QUANTILES[col][n]).sum()
-                elif n < 9:
-                    bucket_count = (
-                        total_count
-                        - count_sum
-                        - ((DATA[table][col] >= PSI_QUANTILES[col][n]).sum())
-                    )
-                elif n == 9:
-                    bucket_count = total_count - count_sum
-                count_sum += bucket_count
-                PSI_BUCKET_FRAC[table, col, n] = bucket_count / total_count
-    return PSI_BUCKET_FRAC, col, n
-
-
-def plot_hist(PSI_BUCKET_FRAC, bins):
-    bin_table_psi = pd.DataFrame(PSI_BUCKET_FRAC)
-    save_fig = []
-    for i in range(len(bin_table_psi)):
+    # Create distribution plots
+    figures = []
+    for feature in feature_columns:
+        reference_dist = [distributions[("reference", feature, b)] for b in bins]
+        monitoring_dist = [distributions[("monitoring", feature, b)] for b in bins]
+        fig = create_distribution_plot(feature, reference_dist, monitoring_dist, bins)
+        figures.append(fig)
 
-        x = pd.DataFrame(
-            bin_table_psi.iloc[i]["test"].items(),
-            columns=["Bin", "Population % Reference"],
-        )
-        y = pd.DataFrame(
-            bin_table_psi.iloc[i]["train"].items(),
-            columns=["Bin", "Population % Monitoring"],
-        )
-        xy = x.merge(y, on="Bin")
-        xy.index = xy["Bin"]
-        xy = xy.drop(columns="Bin", axis=1)
-        feature_name = bin_table_psi.index[i]
-
-        n = len(bins)
-        r = np.arange(n)
-        width = 0.25
-
-        fig = plt.figure()
-
-        plt.bar(
-            r,
-            xy["Population % Reference"],
-            color="b",
-            width=width,
-            edgecolor="black",
-            label="Reference {0}".format(feature_name),
-        )
-        plt.bar(
-            r + width,
-            xy["Population % Monitoring"],
-            color="g",
-            width=width,
-            edgecolor="black",
-            label="Monitoring {0}".format(feature_name),
-        )
+    # Calculate overall pass/fail
+    pass_fail_bool = (psi_df["Pass/Fail"] == "Pass").all()
 
-        plt.xlabel("Bin")
-        plt.ylabel("Population %")
-        plt.title("Histogram of Population Differences {0}".format(feature_name))
-        plt.legend()
-        plt.tight_layout()
-        plt.close()
-        save_fig.append(fig)
-    return save_fig
+    return ({"PSI Scores": psi_df}, *figures, pass_fail_bool)

validmind/tests/ongoing_monitoring/PredictionAcrossEachFeature.py

@@ -53,30 +53,25 @@ def PredictionAcrossEachFeature(datasets, model):
     observed during the training of the model.
     """
 
-    df_reference = datasets[0]._df
-    df_monitoring = datasets[1]._df
+    y_prob_reference = datasets[0].y_prob(model)
+    y_prob_monitoring = datasets[1].y_prob(model)
 
     figures_to_save = []
-    for column in df_reference:
-        prediction_prob_column = f"{model.input_id}_probabilities"
-        prediction_column = f"{model.input_id}_prediction"
-        if column == prediction_prob_column or column == prediction_column:
-            pass
-        else:
-            fig, axs = plt.subplots(1, 2, figsize=(20, 10), sharey="row")
-
-            ax1, ax2 = axs
-
-            ax1.scatter(df_reference[column], df_reference[prediction_prob_column])
-            ax2.scatter(df_monitoring[column], df_monitoring[prediction_prob_column])
-
-            ax1.set_title("Reference")
-            ax1.set_xlabel(column)
-            ax1.set_ylabel("Prediction Value")
-
-            ax2.set_title("Monitoring")
-            ax2.set_xlabel(column)
-            figures_to_save.append(fig)
-            plt.close()
+    for column in datasets[0].feature_columns:
+        fig, axs = plt.subplots(1, 2, figsize=(20, 10), sharey="row")
+
+        ax1, ax2 = axs
+
+        ax1.scatter(datasets[0].df[column], y_prob_reference)
+        ax2.scatter(datasets[1].df[column], y_prob_monitoring)
+
+        ax1.set_title("Reference")
+        ax1.set_xlabel(column)
+        ax1.set_ylabel("Prediction Value")
+
+        ax2.set_title("Monitoring")
+        ax2.set_xlabel(column)
+        figures_to_save.append(fig)
+        plt.close()
 
     return tuple(figures_to_save)