validmind 2.13.6__tar.gz → 2.13.7__tar.gz

{validmind-2.13.6 → validmind-2.13.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: validmind
-Version: 2.13.6
+Version: 2.13.7
 Summary: ValidMind Library
 Author-email: Andres Rodriguez <andres@validmind.ai>, Juan Martinez <juan@validmind.ai>, Anil Sorathiya <anil@validmind.ai>, Luis Pallares <luis@validmind.ai>, John Walz <john@validmind.ai>
 License:                          DUAL LICENSE NOTICE

{validmind-2.13.6 → validmind-2.13.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "validmind"
-version = "2.13.6"
+version = "2.13.7"
 description = "ValidMind Library"
 readme = "README.pypi.md"
 requires-python = ">=3.9,<3.15"

validmind-2.13.7/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.13.7"

{validmind-2.13.6 → validmind-2.13.7}/validmind/tests/model_validation/sklearn/CalibrationCurve.py

@@ -4,10 +4,12 @@
 
 from typing import Tuple
 
+import numpy as np
 import plotly.graph_objects as go
 from sklearn.calibration import calibration_curve
 
 from validmind import tags, tasks
+from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 from validmind.vm_models.result import RawData
 
@@ -70,6 +72,14 @@ def CalibrationCurve(
     - Assumes bin boundaries are appropriate for the problem
     - May be affected by class imbalance
     """
+    # Binary-only by design: sklearn's calibration_curve raises a cryptic
+    # "pos_label is not specified" error on multiclass targets, so skip cleanly
+    # like ROCCurve/PrecisionRecallCurve rather than crashing.
+    if len(np.unique(dataset.y)) > 2:
+        raise SkipTestError(
+            "Calibration Curve is only supported for binary classification models"
+        )
+
     prob_true, prob_pred = calibration_curve(
         dataset.y, dataset.y_prob(model), n_bins=n_bins
     )

{validmind-2.13.6 → validmind-2.13.7}/validmind/tests/model_validation/sklearn/ConfusionMatrix.py

@@ -72,15 +72,20 @@ def ConfusionMatrix(
     - It mainly serves as a descriptive tool and does not offer the capability for statistical hypothesis testing.
     - Risks of misinterpretation exist because the matrix doesn't directly provide precision, recall, or F1-score data.
     These metrics have to be computed separately.
+    - The `threshold` parameter only applies to binary classification (it splits a single positive-class probability
+    into two classes). For multiclass targets the model's argmax class predictions are used and `threshold` is ignored.
     """
-    # Get predictions using threshold for binary classification if possible
-    if hasattr(model.model, "predict_proba"):
+    # The `threshold` only has a meaning for binary classification (it splits a
+    # single positive-class probability into two classes). For multiclass we use
+    # the model's argmax class predictions directly, since thresholding a
+    # single probability column would silently produce wrong labels.
+    n_classes = len(np.unique(dataset.y))
+    if n_classes == 2 and hasattr(model.model, "predict_proba"):
         y_prob = dataset.y_prob(model)
         # Handle both 1D and 2D probability arrays
         if y_prob.ndim == 2:
-            y_pred = (y_prob[:, 1] > threshold).astype(int)
-        else:
-            y_pred = (y_prob > threshold).astype(int)
+            y_prob = y_prob[:, 1]
+        y_pred = (y_prob > threshold).astype(int)
     else:
         y_pred = dataset.y_pred(model)
 

{validmind-2.13.6 → validmind-2.13.7}/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py

@@ -15,6 +15,8 @@ from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
+from ._diagnosis_metrics import bind_averaging, full_labels, multiclass_auc
+
 logger = get_logger(__name__)
 
 # TODO: A couple of improvements here could be to:
@@ -96,6 +98,10 @@ def _compute_metrics(
     feature_column: str,
     metric: str,
     is_classification: bool,
+    average: str = None,
+    pos_label=None,
+    labels: list = None,
+    is_multiclass: bool = False,
 ) -> None:
     results["slice"].append(str(region))
     results["shape"].append(df_region.shape[0])
@@ -115,11 +121,25 @@ def _compute_metrics(
         if len(np.unique(y_true)) == 1:
             return results[metric].append(0)
 
+        # The library retains only a single probability column, so a
+        # probability-based multiclass ROC AUC is not possible; fall back to the
+        # label-binarize convention used elsewhere for multiclass targets.
+        if is_multiclass:
+            return results[metric].append(
+                multiclass_auc(y_true, df_region[pred_column].values, labels)
+            )
+
         return results[metric].append(
             metric_func(y_true, df_region[prob_column].values)
         )
 
-    return results[metric].append(metric_func(y_true, df_region[pred_column].values))
+    # Bind averaging so precision/recall/F1 handle multiclass targets and binary
+    # targets encoded outside {0, 1}; accuracy and regression metrics pass through.
+    return results[metric].append(
+        bind_averaging(metric_func, average, pos_label)(
+            y_true, df_region[pred_column].values
+        )
+    )
 
 
 def _plot_overfit_regions(
@@ -253,6 +273,18 @@ def OverfitDiagnosis(
         train_df[prob_column] = datasets[0].y_prob(model)
         test_df[prob_column] = datasets[1].y_prob(model)
 
+    # Resolve the label space once so every feature slice is scored consistently:
+    # multiclass targets use macro averaging (and label-binarize AUC), binary
+    # targets keep the positive label so non-{0, 1} encodings don't break.
+    if is_classification:
+        labels = full_labels(datasets, model)
+        is_multiclass = len(labels) > 2
+        average, pos_label = (
+            ("macro", None) if is_multiclass else ("binary", labels[-1])
+        )
+    else:
+        labels, is_multiclass, average, pos_label = None, False, None, None
+
     test_results = []
     figures = []
     results_headers = ["slice", "shape", "feature", metric]
@@ -277,6 +309,10 @@ def OverfitDiagnosis(
                 pred_column=pred_column,
                 metric=metric,
                 is_classification=is_classification,
+                average=average,
+                pos_label=pos_label,
+                labels=labels,
+                is_multiclass=is_multiclass,
             )
             df_test_region = test_df[
                 (test_df[feature_column] > region.left)
@@ -292,6 +328,10 @@ def OverfitDiagnosis(
                 pred_column=pred_column,
                 metric=metric,
                 is_classification=is_classification,
+                average=average,
+                pos_label=pos_label,
+                labels=labels,
+                is_multiclass=is_multiclass,
             )
 
         results = _prepare_results(results_train, results_test, metric)

{validmind-2.13.6 → validmind-2.13.7}/validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py

@@ -7,6 +7,7 @@ from typing import Dict, List, Tuple
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
+from plotly.subplots import make_subplots
 
 from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
@@ -15,6 +16,8 @@ from validmind.vm_models import VMDataset, VMModel
 
 logger = get_logger(__name__)
 
+_PSI_PALETTE = ["#DE257E", "#1F77B4", "#2CA02C", "#FF7F0E", "#9467BD", "#8C564B"]
+
 
 def calculate_psi(score_initial, score_new, num_bins=10, mode="fixed"):
     """
@@ -76,6 +79,152 @@ def calculate_psi(score_initial, score_new, num_bins=10, mode="fixed"):
     return psi_df.to_dict(orient="records")
 
 
+def _psi_table_rows(psi_results):
+    """Append the summed 'Total' row and format PSI records as table rows."""
+    total_psi = {
+        key: sum(d.get(key, 0) for d in psi_results)
+        for key in psi_results[0].keys()
+        if isinstance(psi_results[0][key], (int, float))
+    }
+    rows_with_total = psi_results + [total_psi]
+
+    table_rows = [
+        {
+            "Bin": (
+                i if i < (len(rows_with_total) - 1) else "Total"
+            ),  # The last bin is the "Total" bin
+            "Count Initial": values["initial"],
+            "Percent Initial (%)": values["percent_initial"] * 100,
+            "Count New": values["new"],
+            "Percent New (%)": values["percent_new"] * 100,
+            "PSI": values["psi"],
+        }
+        for i, values in enumerate(rows_with_total)
+    ]
+    return rows_with_total, table_rows
+
+
+def _multiclass_psi(datasets, model, classes, num_bins, mode):
+    """One-vs-rest PSI for a multiclass model.
+
+    PSI needs a 1-D score distribution to compare across the two datasets. The
+    stored single probability column cannot represent every class, so we ask the
+    underlying estimator for the full per-class probability matrix (mirroring the
+    ROC/PR curve tests). Models without a usable ``predict_proba`` (metadata-only
+    / precomputed single-column predictions) are skipped rather than crashed.
+    """
+    raw_model = getattr(model, "model", None)
+    proba_fn = getattr(raw_model, "predict_proba", None)
+    if not callable(proba_fn):
+        raise SkipTestError(
+            "Multiclass Population Stability Index requires per-class "
+            "probabilities from the underlying model's predict_proba, which is "
+            "not available for this model (e.g. metadata-only / precomputed "
+            "predictions). Skipping."
+        )
+    try:
+        prob_initial = np.asarray(proba_fn(datasets[0].x_df()))
+        prob_new = np.asarray(proba_fn(datasets[1].x_df()))
+    except Exception as e:
+        raise SkipTestError(
+            "Multiclass Population Stability Index could not compute per-class "
+            f"probabilities ({type(e).__name__}). Skipping."
+        ) from e
+
+    n_classes = len(classes)
+    for prob in (prob_initial, prob_new):
+        if prob.ndim != 2 or prob.shape[1] != n_classes:
+            raise SkipTestError(
+                "Multiclass Population Stability Index requires a per-class "
+                f"probability matrix with one column per class (got shape "
+                f"{getattr(prob, 'shape', None)} for {n_classes} classes). Skipping."
+            )
+
+    # predict_proba columns are ordered by sorted class label == np.unique.
+    fig = make_subplots(
+        rows=n_classes,
+        cols=1,
+        specs=[[{"secondary_y": True}] for _ in range(n_classes)],
+        subplot_titles=[f"Class {cls}" for cls in classes],
+        vertical_spacing=0.08,
+    )
+
+    tables = {}
+    raw_per_class = {}
+    for i, cls in enumerate(classes):
+        psi_results = calculate_psi(
+            prob_initial[:, i].copy(),
+            prob_new[:, i].copy(),
+            num_bins=num_bins,
+            mode=mode,
+        )
+        x = list(range(len(psi_results)))
+        color = _PSI_PALETTE[i % len(_PSI_PALETTE)]
+        fig.add_trace(
+            go.Bar(
+                x=x,
+                y=[d["percent_initial"] for d in psi_results],
+                name="Initial",
+                marker=dict(color="#DE257E"),
+                showlegend=i == 0,
+                legendgroup="initial",
+            ),
+            row=i + 1,
+            col=1,
+            secondary_y=False,
+        )
+        fig.add_trace(
+            go.Bar(
+                x=x,
+                y=[d["percent_new"] for d in psi_results],
+                name="New",
+                marker=dict(color="#E8B1F8"),
+                showlegend=i == 0,
+                legendgroup="new",
+            ),
+            row=i + 1,
+            col=1,
+            secondary_y=False,
+        )
+        fig.add_trace(
+            go.Scatter(
+                x=x,
+                y=[d["psi"] for d in psi_results],
+                name="PSI",
+                line=dict(color=color),
+                showlegend=i == 0,
+                legendgroup="psi",
+            ),
+            row=i + 1,
+            col=1,
+            secondary_y=True,
+        )
+
+        rows_with_total, table_rows = _psi_table_rows(psi_results)
+        table_title = (
+            f"Population Stability Index for Class {cls} "
+            f"({datasets[0].input_id} vs {datasets[1].input_id})"
+        )
+        tables[table_title] = table_rows
+        raw_per_class[str(cls)] = rows_with_total
+
+    fig.update_layout(
+        title="Population Stability Index (PSI) — one-vs-rest per class",
+        barmode="group",
+        height=300 * n_classes,
+    )
+
+    return (
+        tables,
+        fig,
+        RawData(
+            psi_raw=raw_per_class,
+            model=model.input_id,
+            datasets=[datasets[0].input_id, datasets[1].input_id],
+        ),
+    )
+
+
 @tags(
     "sklearn", "binary_classification", "multiclass_classification", "model_performance"
 )
@@ -132,10 +281,18 @@ def PopulationStabilityIndex(
     lead to misinterpretations. Any changes in PSI could be due to shifts in the model (model drift), changes in the
     relationships between features and the target variable (concept drift), or both. However, distinguishing between
     these causes is non-trivial.
+    - For multiclass models the PSI is computed one-vs-rest (one table/plot per class), which requires per-class
+    probabilities from the model's `predict_proba`. Models that cannot produce a full per-class probability matrix
+    (e.g. metadata-only models, or predictions supplied as a single precomputed probability column) are skipped for
+    the multiclass case.
     """
     if model.library in ["statsmodels", "pytorch", "catboost"]:
         raise SkipTestError(f"Skiping PSI for {model.library} models")
 
+    classes = np.unique(datasets[0].y)
+    if len(classes) > 2:
+        return _multiclass_psi(datasets, model, classes, num_bins, mode)
+
     psi_results = calculate_psi(
         datasets[0].y_prob(model).copy(),
         datasets[1].y_prob(model).copy(),
@@ -182,35 +339,16 @@ def PopulationStabilityIndex(
         ),
     )
 
-    # sum up the PSI values to get the total values
-    total_psi = {
-        key: sum(d.get(key, 0) for d in psi_results)
-        for key in psi_results[0].keys()
-        if isinstance(psi_results[0][key], (int, float))
-    }
-    psi_results.append(total_psi)
+    # sum up the PSI values to get the total values and format the table rows
+    rows_with_total, table_rows = _psi_table_rows(psi_results)
 
     table_title = f"Population Stability Index for {datasets[0].input_id} and {datasets[1].input_id} Datasets"
 
     return (
-        {
-            table_title: [
-                {
-                    "Bin": (
-                        i if i < (len(psi_results) - 1) else "Total"
-                    ),  # The last bin is the "Total" bin
-                    "Count Initial": values["initial"],
-                    "Percent Initial (%)": values["percent_initial"] * 100,
-                    "Count New": values["new"],
-                    "Percent New (%)": values["percent_new"] * 100,
-                    "PSI": values["psi"],
-                }
-                for i, values in enumerate(psi_results)
-            ],
-        },
+        {table_title: table_rows},
         fig,
         RawData(
-            psi_raw=psi_results,
+            psi_raw=rows_with_total,
             model=model.input_id,
             datasets=[datasets[0].input_id, datasets[1].input_id],
         ),

validmind-2.13.7/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py

@@ -0,0 +1,204 @@
+# Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+# Refer to the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import Tuple
+
+import numpy as np
+import plotly.graph_objects as go
+from sklearn.metrics import average_precision_score, precision_recall_curve
+from sklearn.preprocessing import label_binarize
+
+from validmind import RawData, tags, tasks
+from validmind.errors import SkipTestError
+from validmind.models import FoundationModel
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tags(
+    "sklearn",
+    "binary_classification",
+    "multiclass_classification",
+    "model_performance",
+    "visualization",
+)
+@tasks("classification", "text_classification")
+def PrecisionRecallCurve(
+    model: VMModel, dataset: VMDataset
+) -> Tuple[go.Figure, RawData]:
+    """
+    Evaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve.
+
+    ### Purpose
+
+    The Precision Recall Curve metric is intended to evaluate the trade-off between precision and recall in
+    classification models, particularly binary classification models. It assesses the model's capacity to produce
+    accurate results (high precision), as well as its ability to capture a majority of all positive instances (high
+    recall).
+
+    ### Test Mechanism
+
+    The test extracts ground truth labels and prediction probabilities from the model's test dataset. It applies the
+    `precision_recall_curve` method from the sklearn metrics module to these extracted labels and predictions, which
+    computes a precision-recall pair for each possible threshold. This calculation results in an array of precision and
+    recall scores that can be plotted against each other to form the Precision-Recall Curve. This curve is then
+    visually represented by using Plotly's scatter plot.
+
+    ### Signs of High Risk
+
+    - A lower area under the Precision-Recall Curve signifies high risk.
+    - This corresponds to a model yielding a high amount of false positives (low precision) and/or false negatives (low
+    recall).
+    - If the curve is closer to the bottom left of the plot, rather than being closer to the top right corner, it can
+    be a sign of high risk.
+
+    ### Strengths
+
+    - This metric aptly represents the balance between precision (minimizing false positives) and recall (minimizing
+    false negatives), which is especially critical in scenarios where both values are significant.
+    - Through the graphic representation, it enables an intuitive understanding of the model's performance across
+    different threshold levels.
+
+    ### Limitations
+
+    - For multiclass models the curve is computed one-vs-rest (one curve per class plus a micro-average), which
+    requires per-class probabilities from the model's `predict_proba`. Models that cannot produce a full per-class
+    probability matrix (e.g. Foundation/metadata-only models, or predictions supplied as a single precomputed
+    probability column) are skipped for the multiclass case.
+    - It may not fully represent the overall accuracy of the model if the cost of false positives and false negatives
+    are extremely different, or if the dataset is heavily imbalanced.
+    """
+    if isinstance(model, FoundationModel):
+        raise SkipTestError("Skipping PrecisionRecallCurve for Foundation models")
+
+    y_true = dataset.y
+    classes = np.unique(y_true)
+
+    if len(classes) > 2:
+        return _multiclass_pr_curve(model, dataset, classes)
+
+    precision, recall, _ = precision_recall_curve(y_true, dataset.y_prob(model))
+
+    fig = go.Figure(
+        data=[
+            go.Scatter(
+                x=recall,
+                y=precision,
+                mode="lines",
+                name="Precision-Recall Curve",
+                line=dict(color="#DE257E"),
+            )
+        ],
+        layout=go.Layout(
+            title="Precision-Recall Curve",
+            xaxis=dict(title="Recall"),
+            yaxis=dict(title="Precision"),
+        ),
+    )
+
+    return fig, RawData(
+        precision=precision,
+        recall=recall,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )
+
+
+def _multiclass_pr_curve(
+    model: VMModel, dataset: VMDataset, classes: np.ndarray
+) -> Tuple[go.Figure, RawData]:
+    """One-vs-rest precision-recall curves for a multiclass model.
+
+    Needs the full per-class probability matrix, which the stored single
+    probability column cannot provide, so we ask the model for it directly.
+    Models without a usable ``predict_proba`` (Foundation/metadata-only,
+    precomputed single-column probabilities) are skipped rather than crashed.
+    """
+    # The VMModel wrapper's predict_proba is binary-only (it returns just the
+    # positive-class column), so reach the underlying estimator for the full
+    # per-class probability matrix.
+    raw_model = getattr(model, "model", None)
+    proba_fn = getattr(raw_model, "predict_proba", None)
+    if not callable(proba_fn):
+        raise SkipTestError(
+            "Multiclass Precision-Recall Curve requires per-class probabilities "
+            "from the underlying model's predict_proba, which is not available "
+            "for this model (e.g. Foundation / metadata-only / precomputed "
+            "predictions). Skipping."
+        )
+    try:
+        y_prob = np.asarray(proba_fn(dataset.x_df()))
+    except Exception as e:
+        raise SkipTestError(
+            "Multiclass Precision-Recall Curve could not compute per-class "
+            f"probabilities ({type(e).__name__}). Skipping."
+        ) from e
+
+    n_classes = len(classes)
+    if y_prob.ndim != 2 or y_prob.shape[1] != n_classes:
+        raise SkipTestError(
+            "Multiclass Precision-Recall Curve requires a per-class probability "
+            f"matrix with one column per class (got shape "
+            f"{getattr(y_prob, 'shape', None)} for {n_classes} classes). Skipping."
+        )
+
+    # One-hot the true labels in the same class order predict_proba columns use
+    # (sklearn orders predict_proba columns by sorted class label == np.unique).
+    y_bin = label_binarize(dataset.y.flatten(), classes=classes)
+
+    traces = []
+    raw_precision = {}
+    raw_recall = {}
+    raw_ap = {}
+    palette = ["#DE257E", "#1F77B4", "#2CA02C", "#FF7F0E", "#9467BD", "#8C564B"]
+    for i, cls in enumerate(classes):
+        precision, recall, _ = precision_recall_curve(y_bin[:, i], y_prob[:, i])
+        ap = average_precision_score(y_bin[:, i], y_prob[:, i])
+        key = str(cls)
+        raw_precision[key] = precision
+        raw_recall[key] = recall
+        raw_ap[key] = ap
+        traces.append(
+            go.Scatter(
+                x=recall,
+                y=precision,
+                mode="lines",
+                name=f"Class {key} (AP = {ap:.2f})",
+                line=dict(color=palette[i % len(palette)]),
+            )
+        )
+
+    # Micro-average across all one-vs-rest decisions.
+    micro_precision, micro_recall, _ = precision_recall_curve(
+        y_bin.ravel(), y_prob.ravel()
+    )
+    micro_ap = average_precision_score(y_bin, y_prob, average="micro")
+    raw_precision["micro"] = micro_precision
+    raw_recall["micro"] = micro_recall
+    raw_ap["micro"] = micro_ap
+    traces.append(
+        go.Scatter(
+            x=micro_recall,
+            y=micro_precision,
+            mode="lines",
+            name=f"Micro-average (AP = {micro_ap:.2f})",
+            line=dict(color="black", dash="dot"),
+        )
+    )
+
+    fig = go.Figure(
+        data=traces,
+        layout=go.Layout(
+            title=f"Precision-Recall Curve (one-vs-rest) for {model.input_id} on {dataset.input_id}",
+            xaxis=dict(title="Recall"),
+            yaxis=dict(title="Precision"),
+        ),
+    )
+
+    return fig, RawData(
+        precision=raw_precision,
+        recall=raw_recall,
+        average_precision=raw_ap,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )