validmind 2.7.2__py3-none-any.whl → 2.7.5__py3-none-any.whl

validmind/tests/data_validation/ScoreBandDefaultRates.py

@@ -0,0 +1,139 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import pandas as pd
+import numpy as np
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tags("visualization", "credit_risk", "scorecard")
+@tasks("classification")
+def ScoreBandDefaultRates(
+    dataset: VMDataset,
+    model: VMModel,
+    score_column: str = "score",
+    score_bands: list = None,
+):
+    """
+    Analyzes default rates and population distribution across credit score bands.
+
+    ### Purpose
+
+    The Score Band Default Rates test evaluates the discriminatory power of credit scores by analyzing
+    default rates across different score bands. This helps validate score effectiveness, supports
+    policy decisions, and provides insights into portfolio risk distribution.
+
+    ### Test Mechanism
+
+    The test segments the score distribution into bands and calculates key metrics for each band:
+    1. Population count and percentage in each band
+    2. Default rate within each band
+    3. Cumulative statistics across bands
+    The results show how well the scores separate good and bad accounts.
+
+    ### Signs of High Risk
+
+    - Non-monotonic default rates across score bands
+    - Insufficient population in critical score bands
+    - Unexpected default rates for score ranges
+    - High concentration in specific score bands
+    - Similar default rates across adjacent bands
+    - Unstable default rates in key decision bands
+    - Extreme population skewness
+    - Poor risk separation between bands
+
+    ### Strengths
+
+    - Clear view of score effectiveness
+    - Supports policy threshold decisions
+    - Easy to interpret and communicate
+    - Directly links to business decisions
+    - Shows risk segmentation power
+    - Identifies potential score issues
+    - Helps validate scoring model
+    - Supports portfolio monitoring
+
+    ### Limitations
+
+    - Sensitive to band definition choices
+    - May mask within-band variations
+    - Requires sufficient data in each band
+    - Cannot capture non-linear patterns
+    - Point-in-time analysis only
+    - No temporal trend information
+    - Assumes band boundaries are appropriate
+    - May oversimplify risk patterns
+    """
+
+    if score_column not in dataset.df.columns:
+        raise ValueError(
+            f"The required column '{score_column}' is not present in the dataset with input_id {dataset.input_id}"
+        )
+
+    df = dataset._df.copy()
+
+    # Default score bands if none provided
+    if score_bands is None:
+        score_bands = [410, 440, 470]
+
+    # Create band labels
+    band_labels = [
+        f"{score_bands[i]}-{score_bands[i+1]}" for i in range(len(score_bands) - 1)
+    ]
+    band_labels.insert(0, f"<{score_bands[0]}")
+    band_labels.append(f">{score_bands[-1]}")
+
+    # Bin the scores with infinite upper bound
+    df["score_band"] = pd.cut(
+        df[score_column], bins=[-np.inf] + score_bands + [np.inf], labels=band_labels
+    )
+
+    # Calculate min and max scores for the total row
+    min_score = df[score_column].min()
+    max_score = df[score_column].max()
+
+    # Get predicted classes (0/1)
+    y_pred = dataset.y_pred(model)
+
+    # Calculate metrics by band using target_column name
+    results = []
+    for band in band_labels:
+        band_mask = df["score_band"] == band
+        population = band_mask.sum()
+        observed_defaults = df[band_mask][dataset.target_column].sum()
+        predicted_defaults = y_pred[
+            band_mask
+        ].sum()  # Sum of 1s gives number of predicted defaults
+
+        results.append(
+            {
+                "Score Band": band,
+                "Population Count": population,
+                "Population (%)": population / len(df) * 100,
+                "Predicted Default Rate (%)": (
+                    predicted_defaults / population * 100 if population > 0 else 0
+                ),
+                "Observed Default Rate (%)": (
+                    observed_defaults / population * 100 if population > 0 else 0
+                ),
+            }
+        )
+
+    # Add total row
+    total_population = len(df)
+    total_observed = df[dataset.target_column].sum()
+    total_predicted = y_pred.sum()  # Total number of predicted defaults
+
+    results.append(
+        {
+            "Score Band": f"Total ({min_score:.0f}-{max_score:.0f})",
+            "Population Count": total_population,
+            "Population (%)": sum(r["Population (%)"] for r in results),
+            "Predicted Default Rate (%)": total_predicted / total_population * 100,
+            "Observed Default Rate (%)": total_observed / total_population * 100,
+        }
+    )
+
+    return pd.DataFrame(results)

validmind/tests/data_validation/TooManyZeroValues.py

@@ -61,24 +61,25 @@ def TooManyZeroValues(dataset: VMDataset, max_percent_threshold: float = 0.03):
     issues.
     """
     df = dataset.df
-
     table = []
 
     for col in dataset.feature_columns_numeric:
         value_counts = df[col].value_counts()
+        row_count = df.shape[0]
 
         if 0 not in value_counts.index:
             continue
 
         n_zeros = value_counts[0]
-        p_zeros = n_zeros / df.shape[0]
+        p_zeros = (n_zeros / row_count) * 100
 
         table.append(
             {
-                "Column": col,
+                "Variable": col,
+                "Row Count": row_count,
                 "Number of Zero Values": n_zeros,
-                "Percentage of Zero Values (%)": p_zeros * 100,
-                "Pass/Fail": "Pass" if p_zeros < max_percent_threshold else "Fail",
+                "Percentage of Zero Values (%)": p_zeros,
+                "Pass/Fail": ("Pass" if p_zeros < (max_percent_threshold) else "Fail"),
             }
         )
 

validmind/tests/data_validation/UniqueRows.py

@@ -61,7 +61,9 @@ def UniqueRows(dataset: VMDataset, min_percent_threshold: float = 1):
             "Number of Unique Values": unique_rows[col],
             "Percentage of Unique Values (%)": unique_rows[col] / rows * 100,
             "Pass/Fail": (
-                "Pass" if unique_rows[col] / rows >= min_percent_threshold else "Fail"
+                "Pass"
+                if (unique_rows[col] / rows * 100) >= min_percent_threshold
+                else "Fail"
             ),
         }
         for col in unique_rows.index

validmind/tests/decorator.py

@@ -24,6 +24,11 @@ def _get_save_func(func, test_id):
     test library.
     """
 
+    # get og source before its wrapped by the test decorator
+    source = inspect.getsource(func)
+    # remove decorator line
+    source = source.split("\n", 1)[1]
+
     def save(root_folder=".", imports=None):
         parts = test_id.split(".")
 
@@ -41,35 +46,32 @@ def _get_save_func(func, test_id):
 
         full_path = os.path.join(path, f"{test_name}.py")
 
-        source = inspect.getsource(func)
-        # remove decorator line
-        source = source.split("\n", 1)[1]
+        _source = source.replace(f"def {func.__name__}", f"def {test_name}")
+
         if imports:
             imports = "\n".join(imports)
-            source = f"{imports}\n\n\n{source}"
+            _source = f"{imports}\n\n\n{_source}"
+
         # add comment to the top of the file
-        source = f"""
+        _source = f"""
 # Saved from {func.__module__}.{func.__name__}
 # Original Test ID: {test_id}
 # New Test ID: {new_test_id}
 
-{source}
+{_source}
 """
 
-        # ensure that the function name matches the test name
-        source = source.replace(f"def {func.__name__}", f"def {test_name}")
-
         # use black to format the code
         try:
             import black
 
-            source = black.format_str(source, mode=black.FileMode())
+            _source = black.format_str(_source, mode=black.FileMode())
         except ImportError:
             # ignore if not available
             pass
 
         with open(full_path, "w") as file:
-            file.writelines(source)
+            file.writelines(_source)
 
         logger.info(
             f"Saved to {os.path.abspath(full_path)}!"
@@ -119,12 +121,12 @@ def test(func_or_id):
         test_func = load_test(test_id, func, reload=True)
         test_store.register_test(test_id, test_func)
 
-        @wraps(test_func)
-        def wrapper(*args, **kwargs):
-            return test_func(*args, **kwargs)
-
         # special function to allow the function to be saved to a file
-        wrapper.save = _get_save_func(test_func, test_id)
+        save_func = _get_save_func(func, test_id)
+
+        wrapper = wraps(func)(test_func)
+        wrapper.test_id = test_id
+        wrapper.save = save_func
 
         return wrapper
 

validmind/tests/load.py

@@ -191,7 +191,7 @@ def list_tags():
     return list(unique_tags)
 
 
-def list_tasks_and_tags():
+def list_tasks_and_tags(as_json=False):
     """
     List all task types and their associated tags, with one row per task type and
     all tags for a task type in one row.
@@ -205,6 +205,9 @@ def list_tasks_and_tags():
         for task in test.__tasks__:
             task_tags_dict.setdefault(task, set()).update(test.__tags__)
 
+    if as_json:
+        return task_tags_dict
+
     return format_dataframe(
         pd.DataFrame(
             [

validmind/tests/model_validation/sklearn/CalibrationCurve.py

@@ -0,0 +1,116 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from sklearn.calibration import calibration_curve
+import plotly.graph_objects as go
+from validmind import tags, tasks
+from validmind.vm_models import VMModel, VMDataset
+from validmind.vm_models.result import RawData
+
+
+@tags("sklearn", "model_performance", "classification")
+@tasks("classification")
+def CalibrationCurve(model: VMModel, dataset: VMDataset, n_bins: int = 10):
+    """
+    Evaluates the calibration of probability estimates by comparing predicted probabilities against observed
+    frequencies.
+
+    ### Purpose
+
+    The Calibration Curve test assesses how well a model's predicted probabilities align with actual
+    observed frequencies. This is crucial for applications requiring accurate probability estimates,
+    such as risk assessment, decision-making systems, and cost-sensitive applications where probability
+    calibration directly impacts business decisions.
+
+    ### Test Mechanism
+
+    The test uses sklearn's calibration_curve function to:
+    1. Sort predictions into bins based on predicted probabilities
+    2. Calculate the mean predicted probability in each bin
+    3. Compare against the observed frequency of positive cases
+    4. Plot the results against the perfect calibration line (y=x)
+    The resulting curve shows how well the predicted probabilities match empirical probabilities.
+
+    ### Signs of High Risk
+
+    - Significant deviation from the perfect calibration line
+    - Systematic overconfidence (predictions too close to 0 or 1)
+    - Systematic underconfidence (predictions clustered around 0.5)
+    - Empty or sparse bins indicating poor probability coverage
+    - Sharp discontinuities in the calibration curve
+    - Different calibration patterns across different probability ranges
+    - Consistent over/under estimation in critical probability regions
+    - Large confidence intervals in certain probability ranges
+
+    ### Strengths
+
+    - Visual and intuitive interpretation of probability quality
+    - Identifies systematic biases in probability estimates
+    - Supports probability threshold selection
+    - Helps understand model confidence patterns
+    - Applicable across different classification models
+    - Enables comparison between different models
+    - Guides potential need for recalibration
+    - Critical for risk-sensitive applications
+
+    ### Limitations
+
+    - Sensitive to the number of bins chosen
+    - Requires sufficient samples in each bin for reliable estimates
+    - May mask local calibration issues within bins
+    - Does not account for feature-dependent calibration issues
+    - Limited to binary classification problems
+    - Cannot detect all forms of miscalibration
+    - Assumes bin boundaries are appropriate for the problem
+    - May be affected by class imbalance
+    """
+    prob_true, prob_pred = calibration_curve(
+        dataset.y, dataset.y_prob(model), n_bins=n_bins
+    )
+
+    # Create DataFrame for raw data
+    raw_data = RawData(
+        mean_predicted_probability=prob_pred, observed_frequency=prob_true
+    )
+
+    # Create Plotly figure
+    fig = go.Figure()
+
+    # Add perfect calibration line
+    fig.add_trace(
+        go.Scatter(
+            x=[0, 1],
+            y=[0, 1],
+            mode="lines",
+            name="Perfect Calibration",
+            line=dict(dash="dash", color="gray"),
+        )
+    )
+
+    # Add calibration curve
+    fig.add_trace(
+        go.Scatter(
+            x=prob_pred,
+            y=prob_true,
+            mode="lines+markers",
+            name="Model Calibration",
+            line=dict(color="blue"),
+            marker=dict(size=8),
+        )
+    )
+
+    # Update layout
+    fig.update_layout(
+        title="Calibration Curve",
+        xaxis_title="Mean Predicted Probability",
+        yaxis_title="Observed Frequency",
+        xaxis=dict(range=[0, 1]),
+        yaxis=dict(range=[0, 1]),
+        width=800,
+        height=600,
+        showlegend=True,
+        template="plotly_white",
+    )
+
+    return raw_data, fig

validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py

@@ -0,0 +1,261 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+from sklearn.metrics import (
+    roc_curve,
+    precision_recall_curve,
+    confusion_matrix,
+)
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+def find_optimal_threshold(y_true, y_prob, method="youden", target_recall=None):
+    """
+    Find the optimal classification threshold using various methods.
+
+    Args:
+        y_true: True binary labels
+        y_prob: Predicted probabilities
+        method: Method to use for finding optimal threshold
+        target_recall: Required if method='target_recall'
+
+    Returns:
+        dict: Dictionary containing threshold and metrics
+    """
+    # Get ROC and PR curve points
+    fpr, tpr, thresholds_roc = roc_curve(y_true, y_prob)
+    precision, recall, thresholds_pr = precision_recall_curve(y_true, y_prob)
+
+    # Find optimal threshold based on method
+    if method == "naive":
+        optimal_threshold = 0.5
+    elif method == "youden":
+        j_scores = tpr - fpr
+        best_idx = np.argmax(j_scores)
+        optimal_threshold = thresholds_roc[best_idx]
+    elif method == "f1":
+        f1_scores = 2 * (precision * recall) / (precision + recall)
+        best_idx = np.argmax(f1_scores)
+        optimal_threshold = (
+            thresholds_pr[best_idx] if best_idx < len(thresholds_pr) else 1.0
+        )
+    elif method == "precision_recall":
+        diff = abs(precision - recall)
+        best_idx = np.argmin(diff)
+        optimal_threshold = (
+            thresholds_pr[best_idx] if best_idx < len(thresholds_pr) else 1.0
+        )
+    elif method == "target_recall":
+        if target_recall is None:
+            raise ValueError(
+                "target_recall must be specified when method='target_recall'"
+            )
+        idx = np.argmin(abs(recall - target_recall))
+        optimal_threshold = thresholds_pr[idx] if idx < len(thresholds_pr) else 1.0
+    else:
+        raise ValueError(f"Unknown method: {method}")
+
+    # Calculate predictions with optimal threshold
+    y_pred = (y_prob >= optimal_threshold).astype(int)
+
+    # Calculate confusion matrix
+    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
+
+    # Calculate metrics directly
+    metrics = {
+        "method": method,
+        "threshold": optimal_threshold,
+        "precision": tp / (tp + fp) if (tp + fp) > 0 else 0,
+        "recall": tp / (tp + fn) if (tp + fn) > 0 else 0,
+        "f1_score": 2 * tp / (2 * tp + fp + fn) if (2 * tp + fp + fn) > 0 else 0,
+        "accuracy": (tp + tn) / (tp + tn + fp + fn),
+    }
+
+    return metrics
+
+
+@tags("model_validation", "threshold_optimization", "classification_metrics")
+@tasks("classification")
+def ClassifierThresholdOptimization(
+    dataset: VMDataset, model: VMModel, methods=None, target_recall=None
+):
+    """
+    Analyzes and visualizes different threshold optimization methods for binary classification models.
+
+    ### Purpose
+
+    The Classifier Threshold Optimization test identifies optimal decision thresholds using various
+    methods to balance different performance metrics. This helps adapt the model's decision boundary
+    to specific business requirements, such as minimizing false positives in fraud detection or
+    achieving target recall in medical diagnosis.
+
+    ### Test Mechanism
+
+    The test implements multiple threshold optimization methods:
+    1. Youden's J statistic (maximizing sensitivity + specificity - 1)
+    2. F1-score optimization (balancing precision and recall)
+    3. Precision-Recall equality point
+    4. Target recall achievement
+    5. Naive (0.5) threshold
+    For each method, it computes ROC and PR curves, identifies optimal points, and provides
+    comprehensive performance metrics at each threshold.
+
+    ### Signs of High Risk
+
+    - Large discrepancies between different optimization methods
+    - Optimal thresholds far from the default 0.5
+    - Poor performance metrics across all thresholds
+    - Significant gap between achieved and target recall
+    - Unstable thresholds across different methods
+    - Extreme trade-offs between precision and recall
+    - Threshold optimization showing minimal impact
+    - Business metrics not improving with optimization
+
+    ### Strengths
+
+    - Multiple optimization strategies for different needs
+    - Visual and numerical results for comparison
+    - Support for business-driven optimization (target recall)
+    - Comprehensive performance metrics at each threshold
+    - Integration with ROC and PR curves
+    - Handles class imbalance through various metrics
+    - Enables informed threshold selection
+    - Supports cost-sensitive decision making
+
+    ### Limitations
+
+    - Assumes cost of false positives/negatives are known
+    - May need adjustment for highly imbalanced datasets
+    - Threshold might not be stable across different samples
+    - Cannot handle multi-class problems directly
+    - Optimization methods may conflict with business needs
+    - Requires sufficient validation data
+    - May not capture temporal changes in optimal threshold
+    - Single threshold may not be optimal for all subgroups
+
+    Args:
+        dataset: VMDataset containing features and target
+        model: VMModel containing predictions
+        methods: List of methods to compare (default: ['youden', 'f1', 'precision_recall'])
+        target_recall: Target recall value if using 'target_recall' method
+
+    Returns:
+        Dictionary containing:
+            - table: DataFrame comparing different threshold optimization methods
+                    (using weighted averages for precision, recall, and f1)
+            - figure: Plotly figure showing ROC and PR curves with optimal thresholds
+    """
+    # Verify binary classification
+    unique_values = np.unique(dataset.y)
+    if len(unique_values) != 2:
+        raise ValueError("Target variable must be binary")
+
+    if methods is None:
+        methods = ["naive", "youden", "f1", "precision_recall"]
+        if target_recall is not None:
+            methods.append("target_recall")
+
+    y_true = dataset.y
+    y_prob = dataset.y_prob(model)
+
+    # Get curve points for plotting
+    fpr, tpr, thresholds_roc = roc_curve(y_true, y_prob)
+    precision, recall, thresholds_pr = precision_recall_curve(y_true, y_prob)
+
+    # Calculate optimal thresholds and metrics
+    results = []
+    optimal_points = {}
+
+    for method in methods:
+        metrics = find_optimal_threshold(y_true, y_prob, method, target_recall)
+        results.append(metrics)
+
+        # Store optimal points for plotting
+        if method == "youden":
+            idx = np.argmax(tpr - fpr)
+            optimal_points[method] = {
+                "x": fpr[idx],
+                "y": tpr[idx],
+                "threshold": thresholds_roc[idx],
+            }
+        elif method in ["f1", "precision_recall", "target_recall"]:
+            idx = np.argmin(abs(thresholds_pr - metrics["threshold"]))
+            optimal_points[method] = {
+                "x": recall[idx],
+                "y": precision[idx],
+                "threshold": metrics["threshold"],
+            }
+
+    # Create visualization
+    fig = make_subplots(
+        rows=1, cols=2, subplot_titles=("ROC Curve", "Precision-Recall Curve")
+    )
+
+    # Plot ROC curve
+    fig.add_trace(
+        go.Scatter(x=fpr, y=tpr, name="ROC Curve", line=dict(color="blue")),
+        row=1,
+        col=1,
+    )
+
+    # Plot PR curve
+    fig.add_trace(
+        go.Scatter(x=recall, y=precision, name="PR Curve", line=dict(color="green")),
+        row=1,
+        col=2,
+    )
+
+    # Add optimal points
+    colors = {
+        "youden": "red",
+        "f1": "orange",
+        "precision_recall": "purple",
+        "target_recall": "brown",
+    }
+
+    for method, points in optimal_points.items():
+        if method == "youden":
+            fig.add_trace(
+                go.Scatter(
+                    x=[points["x"]],
+                    y=[points["y"]],
+                    name=f'{method} (t={points["threshold"]:.2f})',
+                    mode="markers",
+                    marker=dict(size=10, color=colors[method]),
+                ),
+                row=1,
+                col=1,
+            )
+        else:
+            fig.add_trace(
+                go.Scatter(
+                    x=[points["x"]],
+                    y=[points["y"]],
+                    name=f'{method} (t={points["threshold"]:.2f})',
+                    mode="markers",
+                    marker=dict(size=10, color=colors[method]),
+                ),
+                row=1,
+                col=2,
+            )
+
+    # Update layout
+    fig.update_layout(
+        height=500, title_text="Threshold Optimization Analysis", showlegend=True
+    )
+
+    fig.update_xaxes(title_text="False Positive Rate", row=1, col=1)
+    fig.update_xaxes(title_text="Recall", row=1, col=2)
+    fig.update_yaxes(title_text="True Positive Rate", row=1, col=1)
+    fig.update_yaxes(title_text="Precision", row=1, col=2)
+
+    # Create results table and sort by threshold descending
+    table = pd.DataFrame(results).sort_values("threshold", ascending=False)
+
+    return fig, table

validmind/tests/model_validation/sklearn/ConfusionMatrix.py

@@ -106,6 +106,7 @@ def ConfusionMatrix(dataset: VMDataset, model: VMModel):
         autosize=False,
         width=600,
         height=600,
+        title_text="Confusion Matrix",
     )
 
     fig.add_annotation(