validmind 2.7.5__py3-none-any.whl → 2.7.6__py3-none-any.whl

validmind/tests/ongoing_monitoring/CalibrationCurveDrift.py

@@ -0,0 +1,218 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from sklearn.calibration import calibration_curve
+from typing import List
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tags(
+    "sklearn",
+    "binary_classification",
+    "model_performance",
+    "visualization",
+)
+@tasks("classification", "text_classification")
+def CalibrationCurveDrift(
+    datasets: List[VMDataset],
+    model: VMModel,
+    n_bins: int = 10,
+    drift_pct_threshold: float = 20,
+):
+    """
+    Evaluates changes in probability calibration between reference and monitoring datasets.
+
+    ### Purpose
+
+    The Calibration Curve Drift test is designed to assess changes in the model's probability calibration
+    over time. By comparing calibration curves between reference and monitoring datasets, this test helps
+    identify whether the model's probability estimates remain reliable in production. This is crucial for
+    understanding if the model's risk predictions maintain their intended interpretation and whether
+    recalibration might be necessary.
+
+    ### Test Mechanism
+
+    This test proceeds by generating calibration curves for both reference and monitoring datasets. For each
+    dataset, it bins the predicted probabilities and calculates the actual fraction of positives within each
+    bin. It then compares these values between datasets to identify significant shifts in calibration.
+    The test quantifies drift as percentage changes in both mean predicted probabilities and actual fractions
+    of positives per bin, providing both visual and numerical assessments of calibration stability.
+
+    ### Signs of High Risk
+
+    - Large differences between reference and monitoring calibration curves
+    - Systematic over-estimation or under-estimation in monitoring dataset
+    - Significant drift percentages exceeding the threshold in multiple bins
+    - Changes in calibration concentrated in specific probability ranges
+    - Inconsistent drift patterns across the probability spectrum
+    - Empty or sparse bins indicating insufficient data for reliable comparison
+
+    ### Strengths
+
+    - Provides visual and quantitative assessment of calibration changes
+    - Identifies specific probability ranges where calibration has shifted
+    - Enables early detection of systematic prediction biases
+    - Includes detailed bin-by-bin comparison of calibration metrics
+    - Handles edge cases with insufficient data in certain bins
+    - Supports both binary and probabilistic interpretation of results
+
+    ### Limitations
+
+    - Requires sufficient data in each probability bin for reliable comparison
+    - Sensitive to choice of number of bins and binning strategy
+    - May not capture complex changes in probability distributions
+    - Cannot directly suggest recalibration parameters
+    - Limited to assessing probability calibration aspects
+    - Results may be affected by class imbalance changes
+    """
+
+    # Check for binary classification
+    if len(np.unique(datasets[0].y)) > 2:
+        raise SkipTestError(
+            "Calibration Curve Drift is only supported for binary classification models"
+        )
+
+    # Calculate calibration for reference dataset
+    y_prob_ref = datasets[0].y_prob(model)
+    y_true_ref = datasets[0].y.astype(y_prob_ref.dtype).flatten()
+    prob_true_ref, prob_pred_ref = calibration_curve(
+        y_true_ref, y_prob_ref, n_bins=n_bins, strategy="uniform"
+    )
+
+    # Calculate calibration for monitoring dataset
+    y_prob_mon = datasets[1].y_prob(model)
+    y_true_mon = datasets[1].y.astype(y_prob_mon.dtype).flatten()
+    prob_true_mon, prob_pred_mon = calibration_curve(
+        y_true_mon, y_prob_mon, n_bins=n_bins, strategy="uniform"
+    )
+
+    # Create bin labels
+    bin_edges = np.linspace(0, 1, n_bins + 1)
+    bin_labels = [f"{bin_edges[i]:.1f}-{bin_edges[i+1]:.1f}" for i in range(n_bins)]
+
+    # Create predicted probabilities table
+    pred_metrics = []
+    for i in range(n_bins):
+        ref_val = "no data" if i >= len(prob_pred_ref) else round(prob_pred_ref[i], 3)
+        mon_val = "no data" if i >= len(prob_pred_mon) else round(prob_pred_mon[i], 3)
+
+        pred_metrics.append(
+            {"Bin": bin_labels[i], "Reference": ref_val, "Monitoring": mon_val}
+        )
+
+    pred_df = pd.DataFrame(pred_metrics)
+
+    # Calculate drift only for bins with data
+    mask = (pred_df["Reference"] != "no data") & (pred_df["Monitoring"] != "no data")
+    pred_df["Drift (%)"] = None
+    pred_df.loc[mask, "Drift (%)"] = (
+        (
+            pd.to_numeric(pred_df.loc[mask, "Monitoring"])
+            - pd.to_numeric(pred_df.loc[mask, "Reference"])
+        )
+        / pd.to_numeric(pred_df.loc[mask, "Reference"]).abs()
+        * 100
+    ).round(2)
+
+    pred_df["Pass/Fail"] = None
+    pred_df.loc[mask, "Pass/Fail"] = (
+        pred_df.loc[mask, "Drift (%)"]
+        .abs()
+        .apply(lambda x: "Pass" if x < drift_pct_threshold else "Fail")
+    )
+    pred_df.loc[~mask, "Pass/Fail"] = "N/A"
+
+    # Create fraction of positives table
+    true_metrics = []
+    for i in range(n_bins):
+        ref_val = "no data" if i >= len(prob_true_ref) else round(prob_true_ref[i], 3)
+        mon_val = "no data" if i >= len(prob_true_mon) else round(prob_true_mon[i], 3)
+
+        true_metrics.append(
+            {"Bin": bin_labels[i], "Reference": ref_val, "Monitoring": mon_val}
+        )
+
+    true_df = pd.DataFrame(true_metrics)
+
+    # Calculate drift only for bins with data
+    mask = (true_df["Reference"] != "no data") & (true_df["Monitoring"] != "no data")
+    true_df["Drift (%)"] = None
+    true_df.loc[mask, "Drift (%)"] = (
+        (
+            pd.to_numeric(true_df.loc[mask, "Monitoring"])
+            - pd.to_numeric(true_df.loc[mask, "Reference"])
+        )
+        / pd.to_numeric(true_df.loc[mask, "Reference"]).abs()
+        * 100
+    ).round(2)
+
+    true_df["Pass/Fail"] = None
+    true_df.loc[mask, "Pass/Fail"] = (
+        true_df.loc[mask, "Drift (%)"]
+        .abs()
+        .apply(lambda x: "Pass" if x < drift_pct_threshold else "Fail")
+    )
+    true_df.loc[~mask, "Pass/Fail"] = "N/A"
+
+    # Create figure
+    fig = go.Figure()
+
+    # Add perfect calibration line
+    fig.add_trace(
+        go.Scatter(
+            x=[0, 1],
+            y=[0, 1],
+            mode="lines",
+            name="Perfect Calibration",
+            line=dict(color="grey", dash="dash"),
+        )
+    )
+
+    # Add reference calibration curve
+    fig.add_trace(
+        go.Scatter(
+            x=prob_pred_ref,
+            y=prob_true_ref,
+            mode="lines+markers",
+            name="Reference",
+            line=dict(color="blue", width=2),
+            marker=dict(size=8),
+        )
+    )
+
+    # Add monitoring calibration curve
+    fig.add_trace(
+        go.Scatter(
+            x=prob_pred_mon,
+            y=prob_true_mon,
+            mode="lines+markers",
+            name="Monitoring",
+            line=dict(color="red", width=2),
+            marker=dict(size=8),
+        )
+    )
+
+    fig.update_layout(
+        title="Calibration Curves Comparison",
+        xaxis=dict(title="Mean Predicted Probability", range=[0, 1]),
+        yaxis=dict(title="Fraction of Positives", range=[0, 1]),
+        width=700,
+        height=500,
+    )
+
+    # Calculate overall pass/fail (only for bins with data)
+    pass_fail_bool = (pred_df.loc[mask, "Pass/Fail"] == "Pass").all() and (
+        true_df.loc[mask, "Pass/Fail"] == "Pass"
+    ).all()
+
+    return (
+        fig,
+        {"Mean Predicted Probabilities": pred_df, "Fraction of Positives": true_df},
+        pass_fail_bool,
+    )

validmind/tests/ongoing_monitoring/ClassDiscriminationDrift.py

@@ -0,0 +1,153 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import roc_auc_score
+from sklearn.preprocessing import LabelBinarizer
+from scipy import stats
+from typing import List
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+def multiclass_roc_auc_score(y_test, y_pred, average="macro"):
+    lb = LabelBinarizer()
+    lb.fit(y_test)
+    return roc_auc_score(lb.transform(y_test), lb.transform(y_pred), average=average)
+
+
+def calculate_gini(y_true, y_prob):
+    """Calculate Gini coefficient (2*AUC - 1)"""
+    return 2 * roc_auc_score(y_true, y_prob) - 1
+
+
+def calculate_ks_statistic(y_true, y_prob):
+    """Calculate Kolmogorov-Smirnov statistic"""
+    pos_scores = y_prob[y_true == 1]
+    neg_scores = y_prob[y_true == 0]
+    return stats.ks_2samp(pos_scores, neg_scores).statistic
+
+
+@tags(
+    "sklearn", "binary_classification", "multiclass_classification", "model_performance"
+)
+@tasks("classification", "text_classification")
+def ClassDiscriminationDrift(
+    datasets: List[VMDataset], model: VMModel, drift_pct_threshold=20
+):
+    """
+    Compares classification discrimination metrics between reference and monitoring datasets.
+
+    ### Purpose
+
+    The Class Discrimination Drift test is designed to evaluate changes in the model's discriminative power
+    over time. By comparing key discrimination metrics between reference and monitoring datasets, this test
+    helps identify whether the model maintains its ability to separate classes in production. This is crucial
+    for understanding if the model's predictive power remains stable and whether its decision boundaries
+    continue to effectively distinguish between different classes.
+
+    ### Test Mechanism
+
+    This test proceeds by calculating three key discrimination metrics for both reference and monitoring
+    datasets: ROC AUC (Area Under the Curve), GINI coefficient, and KS (Kolmogorov-Smirnov) statistic.
+    For binary classification, it computes all three metrics. For multiclass problems, it focuses on
+    macro-averaged ROC AUC. The test quantifies drift as percentage changes in these metrics between
+    datasets, providing a comprehensive assessment of discrimination stability.
+
+    ### Signs of High Risk
+
+    - Large drifts in discrimination metrics exceeding the threshold
+    - Significant drops in ROC AUC indicating reduced ranking ability
+    - Decreased GINI coefficients showing diminished separation power
+    - Reduced KS statistics suggesting weaker class distinction
+    - Inconsistent changes across different metrics
+    - Systematic degradation in discriminative performance
+
+    ### Strengths
+
+    - Combines multiple complementary discrimination metrics
+    - Handles both binary and multiclass classification
+    - Provides clear quantitative drift assessment
+    - Enables early detection of model degradation
+    - Includes standardized drift threshold evaluation
+    - Supports comprehensive performance monitoring
+
+    ### Limitations
+
+    - Does not identify root causes of discrimination drift
+    - May be sensitive to changes in class distribution
+    - Cannot suggest optimal decision threshold adjustments
+    - Limited to discrimination aspects of performance
+    - Requires sufficient data for reliable metric calculation
+    - May not capture subtle changes in decision boundaries
+    """
+    # Get predictions and true values
+    y_true_ref = datasets[0].y
+    y_true_mon = datasets[1].y
+
+    metrics = []
+
+    # Handle binary vs multiclass
+    if len(np.unique(y_true_ref)) == 2:
+        # Binary classification
+        y_prob_ref = datasets[0].y_prob(model)
+        y_prob_mon = datasets[1].y_prob(model)
+
+        # ROC AUC
+        roc_auc_ref = roc_auc_score(y_true_ref, y_prob_ref)
+        roc_auc_mon = roc_auc_score(y_true_mon, y_prob_mon)
+        metrics.append(
+            {"Metric": "ROC_AUC", "Reference": roc_auc_ref, "Monitoring": roc_auc_mon}
+        )
+
+        # GINI
+        gini_ref = calculate_gini(y_true_ref, y_prob_ref)
+        gini_mon = calculate_gini(y_true_mon, y_prob_mon)
+        metrics.append(
+            {"Metric": "GINI", "Reference": gini_ref, "Monitoring": gini_mon}
+        )
+
+        # KS Statistic
+        ks_ref = calculate_ks_statistic(y_true_ref, y_prob_ref)
+        ks_mon = calculate_ks_statistic(y_true_mon, y_prob_mon)
+        metrics.append(
+            {"Metric": "KS_Statistic", "Reference": ks_ref, "Monitoring": ks_mon}
+        )
+
+    else:
+        # Multiclass
+        y_pred_ref = datasets[0].y_pred(model)
+        y_pred_mon = datasets[1].y_pred(model)
+
+        # Only ROC AUC for multiclass
+        roc_auc_ref = multiclass_roc_auc_score(y_true_ref, y_pred_ref)
+        roc_auc_mon = multiclass_roc_auc_score(y_true_mon, y_pred_mon)
+        metrics.append(
+            {
+                "Metric": "ROC_AUC_Macro",
+                "Reference": roc_auc_ref,
+                "Monitoring": roc_auc_mon,
+            }
+        )
+
+    # Create DataFrame
+    df = pd.DataFrame(metrics)
+
+    # Calculate drift percentage with direction
+    df["Drift (%)"] = (
+        (df["Monitoring"] - df["Reference"]) / df["Reference"].abs() * 100
+    ).round(2)
+
+    # Add Pass/Fail column based on absolute drift
+    df["Pass/Fail"] = (
+        df["Drift (%)"]
+        .abs()
+        .apply(lambda x: "Pass" if x < drift_pct_threshold else "Fail")
+    )
+
+    # Calculate overall pass/fail
+    pass_fail_bool = (df["Pass/Fail"] == "Pass").all()
+
+    return ({"Classification Discrimination Metrics": df}, pass_fail_bool)

validmind/tests/ongoing_monitoring/ClassImbalanceDrift.py

@@ -0,0 +1,144 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+import plotly.graph_objs as go
+from typing import List
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset
+from validmind.errors import SkipTestError
+
+
+@tags("tabular_data", "binary_classification", "multiclass_classification")
+@tasks("classification")
+def ClassImbalanceDrift(
+    datasets: List[VMDataset],
+    drift_pct_threshold: float = 5.0,
+    title: str = "Class Distribution Drift",
+):
+    """
+    Evaluates drift in class distribution between reference and monitoring datasets.
+
+    ### Purpose
+
+    The Class Imbalance Drift test is designed to detect changes in the distribution of target classes
+    over time. By comparing class proportions between reference and monitoring datasets, this test helps
+    identify whether the population structure remains stable in production. This is crucial for
+    understanding if the model continues to operate under similar class distribution assumptions and
+    whether retraining might be necessary due to significant shifts in class balance.
+
+    ### Test Mechanism
+
+    This test proceeds by calculating class percentages for both reference and monitoring datasets.
+    It computes the proportion of each class and quantifies drift as the percentage difference in these
+    proportions between datasets. The test provides both visual and numerical comparisons of class
+    distributions, with special attention to changes that exceed the specified drift threshold.
+    Population stability is assessed on a class-by-class basis.
+
+    ### Signs of High Risk
+
+    - Large shifts in class proportions exceeding the threshold
+    - Systematic changes affecting multiple classes
+    - Appearance of new classes or disappearance of existing ones
+    - Significant changes in minority class representation
+    - Reversal of majority-minority class relationships
+    - Unexpected changes in class ratios
+
+    ### Strengths
+
+    - Provides clear visualization of distribution changes
+    - Identifies specific classes experiencing drift
+    - Enables early detection of population shifts
+    - Includes standardized drift threshold evaluation
+    - Supports both binary and multiclass problems
+    - Maintains interpretable percentage-based metrics
+
+    ### Limitations
+
+    - Does not account for feature distribution changes
+    - Cannot identify root causes of class drift
+    - May be sensitive to small sample sizes
+    - Limited to target variable distribution only
+    - Requires sufficient samples per class
+    - May not capture subtle distribution changes
+    """
+    # Validate inputs
+    if not datasets[0].target_column or not datasets[1].target_column:
+        raise SkipTestError("No target column provided")
+
+    # Calculate class distributions
+    ref_dist = (
+        datasets[0].df[datasets[0].target_column].value_counts(normalize=True) * 100
+    )
+    mon_dist = (
+        datasets[1].df[datasets[1].target_column].value_counts(normalize=True) * 100
+    )
+
+    # Get all unique classes
+    all_classes = sorted(set(ref_dist.index) | set(mon_dist.index))
+
+    if len(all_classes) > 10:
+        raise SkipTestError("Skipping target column with more than 10 classes")
+
+    # Create comparison table
+    rows = []
+    all_passed = True
+
+    for class_label in all_classes:
+        ref_percent = ref_dist.get(class_label, 0)
+        mon_percent = mon_dist.get(class_label, 0)
+
+        # Calculate drift (preserving sign)
+        drift = mon_percent - ref_percent
+        passed = abs(drift) < drift_pct_threshold
+        all_passed &= passed
+
+        rows.append(
+            {
+                datasets[0].target_column: class_label,
+                "Reference (%)": round(ref_percent, 4),
+                "Monitoring (%)": round(mon_percent, 4),
+                "Drift (%)": round(drift, 4),
+                "Pass/Fail": "Pass" if passed else "Fail",
+            }
+        )
+
+    comparison_df = pd.DataFrame(rows)
+
+    # Create named tables dictionary
+    tables = {"Class Distribution (%)": comparison_df}
+
+    # Create visualization
+    fig = go.Figure()
+
+    # Add reference distribution bar
+    fig.add_trace(
+        go.Bar(
+            name="Reference",
+            x=[str(c) for c in all_classes],
+            y=comparison_df["Reference (%)"],
+            marker_color="rgba(31, 119, 180, 0.8)",  # Blue with 0.8 opacity
+        )
+    )
+
+    # Add monitoring distribution bar
+    fig.add_trace(
+        go.Bar(
+            name="Monitoring",
+            x=[str(c) for c in all_classes],
+            y=comparison_df["Monitoring (%)"],
+            marker_color="rgba(255, 127, 14, 0.8)",  # Orange with 0.8 opacity
+        )
+    )
+
+    # Update layout
+    fig.update_layout(
+        title=title,
+        xaxis_title="Class",
+        yaxis_title="Percentage (%)",
+        barmode="group",
+        showlegend=True,
+    )
+
+    return fig, tables, all_passed

validmind/tests/ongoing_monitoring/ClassificationAccuracyDrift.py

@@ -0,0 +1,146 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import classification_report
+from typing import List
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tags(
+    "sklearn", "binary_classification", "multiclass_classification", "model_performance"
+)
+@tasks("classification", "text_classification")
+def ClassificationAccuracyDrift(
+    datasets: List[VMDataset], model: VMModel, drift_pct_threshold=20
+):
+    """
+    Compares classification accuracy metrics between reference and monitoring datasets.
+
+    ### Purpose
+
+    The Classification Accuracy Drift test is designed to evaluate changes in the model's predictive accuracy
+    over time. By comparing key accuracy metrics between reference and monitoring datasets, this test helps
+    identify whether the model maintains its performance levels in production. This is crucial for
+    understanding if the model's predictions remain reliable and whether its overall effectiveness has
+    degraded significantly.
+
+    ### Test Mechanism
+
+    This test proceeds by calculating comprehensive accuracy metrics for both reference and monitoring
+    datasets. It computes overall accuracy, per-label precision, recall, and F1 scores, as well as
+    macro-averaged metrics. The test quantifies drift as percentage changes in these metrics between
+    datasets, providing both granular and aggregate views of accuracy changes. Special attention is paid
+    to per-label performance to identify class-specific degradation.
+
+    ### Signs of High Risk
+
+    - Large drifts in accuracy metrics exceeding the threshold
+    - Inconsistent changes across different labels
+    - Significant drops in macro-averaged metrics
+    - Systematic degradation in specific class performance
+    - Unexpected improvements suggesting data quality issues
+    - Divergent trends between precision and recall
+
+    ### Strengths
+
+    - Provides comprehensive accuracy assessment
+    - Identifies class-specific performance changes
+    - Enables early detection of model degradation
+    - Includes both micro and macro perspectives
+    - Supports multi-class classification evaluation
+    - Maintains interpretable drift thresholds
+
+    ### Limitations
+
+    - May be sensitive to class distribution changes
+    - Does not account for prediction confidence
+    - Cannot identify root causes of accuracy drift
+    - Limited to accuracy-based metrics only
+    - Requires sufficient samples per class
+    - May not capture subtle performance changes
+    """
+    # Get predictions and true values
+    y_true_ref = datasets[0].y
+    y_pred_ref = datasets[0].y_pred(model)
+
+    y_true_mon = datasets[1].y
+    y_pred_mon = datasets[1].y_pred(model)
+
+    # Get unique labels from reference dataset
+    labels = np.unique(y_true_ref)
+    labels = sorted(labels.tolist())
+
+    # Calculate classification reports
+    report_ref = classification_report(
+        y_true=y_true_ref,
+        y_pred=y_pred_ref,
+        output_dict=True,
+        zero_division=0,
+    )
+
+    report_mon = classification_report(
+        y_true=y_true_mon,
+        y_pred=y_pred_mon,
+        output_dict=True,
+        zero_division=0,
+    )
+
+    # Create metrics dataframe
+    metrics = []
+
+    # Add accuracy
+    metrics.append(
+        {
+            "Metric": "Accuracy",
+            "Reference": report_ref["accuracy"],
+            "Monitoring": report_mon["accuracy"],
+        }
+    )
+
+    # Add per-label metrics
+    for label in labels:
+        label_str = str(label)
+        for metric in ["precision", "recall", "f1-score"]:
+            metric_name = f"{metric.title()}_{label_str}"
+            metrics.append(
+                {
+                    "Metric": metric_name,
+                    "Reference": report_ref[label_str][metric],
+                    "Monitoring": report_mon[label_str][metric],
+                }
+            )
+
+    # Add macro averages
+    for metric in ["precision", "recall", "f1-score"]:
+        metric_name = f"{metric.title()}_Macro"
+        metrics.append(
+            {
+                "Metric": metric_name,
+                "Reference": report_ref["macro avg"][metric],
+                "Monitoring": report_mon["macro avg"][metric],
+            }
+        )
+
+    # Create DataFrame
+    df = pd.DataFrame(metrics)
+
+    # Calculate drift percentage with direction
+    df["Drift (%)"] = (
+        (df["Monitoring"] - df["Reference"]) / df["Reference"].abs() * 100
+    ).round(2)
+
+    # Add Pass/Fail column based on absolute drift
+    df["Pass/Fail"] = (
+        df["Drift (%)"]
+        .abs()
+        .apply(lambda x: "Pass" if x < drift_pct_threshold else "Fail")
+    )
+
+    # Calculate overall pass/fail
+    pass_fail_bool = (df["Pass/Fail"] == "Pass").all()
+
+    return ({"Classification Accuracy Metrics": df}, pass_fail_bool)