validmind 2.13.6__tar.gz → 2.13.8__tar.gz

{validmind-2.13.6 → validmind-2.13.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: validmind
-Version: 2.13.6
+Version: 2.13.8
 Summary: ValidMind Library
 Author-email: Andres Rodriguez <andres@validmind.ai>, Juan Martinez <juan@validmind.ai>, Anil Sorathiya <anil@validmind.ai>, Luis Pallares <luis@validmind.ai>, John Walz <john@validmind.ai>
 License:                          DUAL LICENSE NOTICE

{validmind-2.13.6 → validmind-2.13.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "validmind"
-version = "2.13.6"
+version = "2.13.8"
 description = "ValidMind Library"
 readme = "README.pypi.md"
 requires-python = ">=3.9,<3.15"

validmind-2.13.8/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.13.8"

{validmind-2.13.6 → validmind-2.13.8}/validmind/api_client.py

@@ -317,7 +317,7 @@ def init(
             Can be set via env ``VM_OIDC_ISSUER``.
         client_id (str, optional): OAuth public client id for device flow. Can be
             set via env ``VM_OIDC_CLIENT_ID``.
-        scope (str, optional): OAuth scopes (default ``openid profile email``).
+        scope (str, optional): OAuth scopes (default ``openid profile email offline_access``).
             Can be set via env ``VM_OIDC_SCOPE``.
         audience (str, optional): Resource / API identifier for the access token
             (e.g. Auth0 API Identifier). Use the same value the ValidMind backend
@@ -392,7 +392,7 @@ def init(
         _api_key = None
         _api_secret = None
         _api_host = resolved_host
-        scope_val = oidc_scope or "openid profile email"
+        scope_val = oidc_scope or "openid profile email offline_access"
         from .credentials_store import normalize_audience
 
         oidc_audience_val = normalize_audience(

{validmind-2.13.6 → validmind-2.13.8}/validmind/tests/model_validation/sklearn/CalibrationCurve.py

@@ -4,10 +4,12 @@
 
 from typing import Tuple
 
+import numpy as np
 import plotly.graph_objects as go
 from sklearn.calibration import calibration_curve
 
 from validmind import tags, tasks
+from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 from validmind.vm_models.result import RawData
 
@@ -70,6 +72,14 @@ def CalibrationCurve(
     - Assumes bin boundaries are appropriate for the problem
     - May be affected by class imbalance
     """
+    # Binary-only by design: sklearn's calibration_curve raises a cryptic
+    # "pos_label is not specified" error on multiclass targets, so skip cleanly
+    # like ROCCurve/PrecisionRecallCurve rather than crashing.
+    if len(np.unique(dataset.y)) > 2:
+        raise SkipTestError(
+            "Calibration Curve is only supported for binary classification models"
+        )
+
     prob_true, prob_pred = calibration_curve(
         dataset.y, dataset.y_prob(model), n_bins=n_bins
     )

{validmind-2.13.6 → validmind-2.13.8}/validmind/tests/model_validation/sklearn/ConfusionMatrix.py

@@ -72,15 +72,20 @@ def ConfusionMatrix(
     - It mainly serves as a descriptive tool and does not offer the capability for statistical hypothesis testing.
     - Risks of misinterpretation exist because the matrix doesn't directly provide precision, recall, or F1-score data.
     These metrics have to be computed separately.
+    - The `threshold` parameter only applies to binary classification (it splits a single positive-class probability
+    into two classes). For multiclass targets the model's argmax class predictions are used and `threshold` is ignored.
     """
-    # Get predictions using threshold for binary classification if possible
-    if hasattr(model.model, "predict_proba"):
+    # The `threshold` only has a meaning for binary classification (it splits a
+    # single positive-class probability into two classes). For multiclass we use
+    # the model's argmax class predictions directly, since thresholding a
+    # single probability column would silently produce wrong labels.
+    n_classes = len(np.unique(dataset.y))
+    if n_classes == 2 and hasattr(model.model, "predict_proba"):
         y_prob = dataset.y_prob(model)
         # Handle both 1D and 2D probability arrays
         if y_prob.ndim == 2:
-            y_pred = (y_prob[:, 1] > threshold).astype(int)
-        else:
-            y_pred = (y_prob > threshold).astype(int)
+            y_prob = y_prob[:, 1]
+        y_pred = (y_prob > threshold).astype(int)
     else:
         y_pred = dataset.y_pred(model)
 

{validmind-2.13.6 → validmind-2.13.8}/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py

@@ -15,6 +15,8 @@ from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
+from ._diagnosis_metrics import bind_averaging, full_labels, multiclass_auc
+
 logger = get_logger(__name__)
 
 # TODO: A couple of improvements here could be to:
@@ -96,6 +98,10 @@ def _compute_metrics(
     feature_column: str,
     metric: str,
     is_classification: bool,
+    average: str = None,
+    pos_label=None,
+    labels: list = None,
+    is_multiclass: bool = False,
 ) -> None:
     results["slice"].append(str(region))
     results["shape"].append(df_region.shape[0])
@@ -115,11 +121,25 @@ def _compute_metrics(
         if len(np.unique(y_true)) == 1:
             return results[metric].append(0)
 
+        # The library retains only a single probability column, so a
+        # probability-based multiclass ROC AUC is not possible; fall back to the
+        # label-binarize convention used elsewhere for multiclass targets.
+        if is_multiclass:
+            return results[metric].append(
+                multiclass_auc(y_true, df_region[pred_column].values, labels)
+            )
+
         return results[metric].append(
             metric_func(y_true, df_region[prob_column].values)
         )
 
-    return results[metric].append(metric_func(y_true, df_region[pred_column].values))
+    # Bind averaging so precision/recall/F1 handle multiclass targets and binary
+    # targets encoded outside {0, 1}; accuracy and regression metrics pass through.
+    return results[metric].append(
+        bind_averaging(metric_func, average, pos_label)(
+            y_true, df_region[pred_column].values
+        )
+    )
 
 
 def _plot_overfit_regions(
@@ -253,6 +273,18 @@ def OverfitDiagnosis(
         train_df[prob_column] = datasets[0].y_prob(model)
         test_df[prob_column] = datasets[1].y_prob(model)
 
+    # Resolve the label space once so every feature slice is scored consistently:
+    # multiclass targets use macro averaging (and label-binarize AUC), binary
+    # targets keep the positive label so non-{0, 1} encodings don't break.
+    if is_classification:
+        labels = full_labels(datasets, model)
+        is_multiclass = len(labels) > 2
+        average, pos_label = (
+            ("macro", None) if is_multiclass else ("binary", labels[-1])
+        )
+    else:
+        labels, is_multiclass, average, pos_label = None, False, None, None
+
     test_results = []
     figures = []
     results_headers = ["slice", "shape", "feature", metric]
@@ -277,6 +309,10 @@ def OverfitDiagnosis(
                 pred_column=pred_column,
                 metric=metric,
                 is_classification=is_classification,
+                average=average,
+                pos_label=pos_label,
+                labels=labels,
+                is_multiclass=is_multiclass,
             )
             df_test_region = test_df[
                 (test_df[feature_column] > region.left)
@@ -292,6 +328,10 @@ def OverfitDiagnosis(
                 pred_column=pred_column,
                 metric=metric,
                 is_classification=is_classification,
+                average=average,
+                pos_label=pos_label,
+                labels=labels,
+                is_multiclass=is_multiclass,
             )
 
         results = _prepare_results(results_train, results_test, metric)

{validmind-2.13.6 → validmind-2.13.8}/validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py

@@ -7,14 +7,19 @@ from typing import Dict, List, Tuple
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
+from plotly.subplots import make_subplots
 
 from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
+from ._multiclass_proba import multiclass_proba, proba_matrix
+
 logger = get_logger(__name__)
 
+_PSI_PALETTE = ["#DE257E", "#1F77B4", "#2CA02C", "#FF7F0E", "#9467BD", "#8C564B"]
+
 
 def calculate_psi(score_initial, score_new, num_bins=10, mode="fixed"):
     """
@@ -76,6 +81,135 @@ def calculate_psi(score_initial, score_new, num_bins=10, mode="fixed"):
     return psi_df.to_dict(orient="records")
 
 
+def _psi_table_rows(psi_results):
+    """Append the summed 'Total' row and format PSI records as table rows."""
+    total_psi = {
+        key: sum(d.get(key, 0) for d in psi_results)
+        for key in psi_results[0].keys()
+        if isinstance(psi_results[0][key], (int, float))
+    }
+    rows_with_total = psi_results + [total_psi]
+
+    table_rows = [
+        {
+            "Bin": (
+                i if i < (len(rows_with_total) - 1) else "Total"
+            ),  # The last bin is the "Total" bin
+            "Count Initial": values["initial"],
+            "Percent Initial (%)": values["percent_initial"] * 100,
+            "Count New": values["new"],
+            "Percent New (%)": values["percent_new"] * 100,
+            "PSI": values["psi"],
+        }
+        for i, values in enumerate(rows_with_total)
+    ]
+    return rows_with_total, table_rows
+
+
+def _multiclass_psi(datasets, model, num_bins, mode):
+    """One-vs-rest PSI for a multiclass model.
+
+    PSI needs a 1-D score distribution to compare across the two datasets. The
+    stored single probability column cannot represent every class, so the shared
+    helper reaches the underlying estimator for the full per-class probability
+    matrix (mirroring the ROC/PR curve tests), aligning columns to the training
+    class order. The initial dataset drives the class alignment and the set of
+    classes actually present; the new dataset's matrix is validated against the
+    same class list. Models that cannot supply a matching matrix are skipped.
+    """
+    aligned = multiclass_proba(model, datasets[0], "Population Stability Index")
+    prob_initial = aligned.y_prob
+    prob_new = proba_matrix(
+        model, datasets[1], aligned.class_list, "Population Stability Index"
+    )
+
+    classes_present = aligned.classes_present
+    n_present = len(classes_present)
+
+    fig = make_subplots(
+        rows=n_present,
+        cols=1,
+        specs=[[{"secondary_y": True}] for _ in range(n_present)],
+        subplot_titles=[f"Class {cls}" for cls in classes_present],
+        vertical_spacing=0.08,
+    )
+
+    tables = {}
+    raw_per_class = {}
+    for plot_i, (i, cls) in enumerate(zip(aligned.present_indices, classes_present)):
+        psi_results = calculate_psi(
+            prob_initial[:, i].copy(),
+            prob_new[:, i].copy(),
+            num_bins=num_bins,
+            mode=mode,
+        )
+        x = list(range(len(psi_results)))
+        color = _PSI_PALETTE[plot_i % len(_PSI_PALETTE)]
+        fig.add_trace(
+            go.Bar(
+                x=x,
+                y=[d["percent_initial"] for d in psi_results],
+                name="Initial",
+                marker=dict(color="#DE257E"),
+                showlegend=plot_i == 0,
+                legendgroup="initial",
+            ),
+            row=plot_i + 1,
+            col=1,
+            secondary_y=False,
+        )
+        fig.add_trace(
+            go.Bar(
+                x=x,
+                y=[d["percent_new"] for d in psi_results],
+                name="New",
+                marker=dict(color="#E8B1F8"),
+                showlegend=plot_i == 0,
+                legendgroup="new",
+            ),
+            row=plot_i + 1,
+            col=1,
+            secondary_y=False,
+        )
+        fig.add_trace(
+            go.Scatter(
+                x=x,
+                y=[d["psi"] for d in psi_results],
+                name="PSI",
+                line=dict(color=color),
+                showlegend=plot_i == 0,
+                legendgroup="psi",
+            ),
+            row=plot_i + 1,
+            col=1,
+            secondary_y=True,
+        )
+
+        rows_with_total, table_rows = _psi_table_rows(psi_results)
+        table_title = (
+            f"Population Stability Index for Class {cls} "
+            f"({datasets[0].input_id} vs {datasets[1].input_id})"
+        )
+        tables[table_title] = table_rows
+        raw_per_class[str(cls)] = rows_with_total
+
+    fig.update_layout(
+        title="Population Stability Index (PSI) — one-vs-rest per class",
+        barmode="group",
+        height=300 * n_present,
+    )
+
+    return (
+        tables,
+        fig,
+        RawData(
+            psi_raw=raw_per_class,
+            model=model.input_id,
+            datasets=[datasets[0].input_id, datasets[1].input_id],
+        ),
+    )
+
+
 @tags(
     "sklearn", "binary_classification", "multiclass_classification", "model_performance"
 )
@@ -132,10 +266,18 @@ def PopulationStabilityIndex(
     lead to misinterpretations. Any changes in PSI could be due to shifts in the model (model drift), changes in the
     relationships between features and the target variable (concept drift), or both. However, distinguishing between
     these causes is non-trivial.
+    - For multiclass models the PSI is computed one-vs-rest (one table/plot per class), which requires per-class
+    probabilities from the model's `predict_proba`. Models that cannot produce a full per-class probability matrix
+    (e.g. metadata-only models, or predictions supplied as a single precomputed probability column) are skipped for
+    the multiclass case.
     """
     if model.library in ["statsmodels", "pytorch", "catboost"]:
         raise SkipTestError(f"Skiping PSI for {model.library} models")
 
+    classes = np.unique(datasets[0].y)
+    if len(classes) > 2:
+        return _multiclass_psi(datasets, model, num_bins, mode)
+
     psi_results = calculate_psi(
         datasets[0].y_prob(model).copy(),
         datasets[1].y_prob(model).copy(),
@@ -182,35 +324,16 @@ def PopulationStabilityIndex(
         ),
     )
 
-    # sum up the PSI values to get the total values
-    total_psi = {
-        key: sum(d.get(key, 0) for d in psi_results)
-        for key in psi_results[0].keys()
-        if isinstance(psi_results[0][key], (int, float))
-    }
-    psi_results.append(total_psi)
+    # sum up the PSI values to get the total values and format the table rows
+    rows_with_total, table_rows = _psi_table_rows(psi_results)
 
     table_title = f"Population Stability Index for {datasets[0].input_id} and {datasets[1].input_id} Datasets"
 
     return (
-        {
-            table_title: [
-                {
-                    "Bin": (
-                        i if i < (len(psi_results) - 1) else "Total"
-                    ),  # The last bin is the "Total" bin
-                    "Count Initial": values["initial"],
-                    "Percent Initial (%)": values["percent_initial"] * 100,
-                    "Count New": values["new"],
-                    "Percent New (%)": values["percent_new"] * 100,
-                    "PSI": values["psi"],
-                }
-                for i, values in enumerate(psi_results)
-            ],
-        },
+        {table_title: table_rows},
         fig,
         RawData(
-            psi_raw=psi_results,
+            psi_raw=rows_with_total,
             model=model.input_id,
             datasets=[datasets[0].input_id, datasets[1].input_id],
         ),

{validmind-2.13.6 → validmind-2.13.8}/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py

@@ -6,15 +6,23 @@ from typing import Tuple
 
 import numpy as np
 import plotly.graph_objects as go
-from sklearn.metrics import precision_recall_curve
+from sklearn.metrics import average_precision_score, precision_recall_curve
 
 from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.models import FoundationModel
 from validmind.vm_models import VMDataset, VMModel
 
+from ._multiclass_proba import multiclass_proba
 
-@tags("sklearn", "binary_classification", "model_performance", "visualization")
+
+@tags(
+    "sklearn",
+    "binary_classification",
+    "multiclass_classification",
+    "model_performance",
+    "visualization",
+)
 @tasks("classification", "text_classification")
 def PrecisionRecallCurve(
     model: VMModel, dataset: VMDataset
@@ -54,8 +62,10 @@ def PrecisionRecallCurve(
 
     ### Limitations
 
-    - This metric is only applicable to binary classification models - it raises errors for multiclass classification
-    models or Foundation models.
+    - For multiclass models the curve is computed one-vs-rest (one curve per class plus a micro-average), which
+    requires per-class probabilities from the model's `predict_proba`. Models that cannot produce a full per-class
+    probability matrix (e.g. Foundation/metadata-only models, or predictions supplied as a single precomputed
+    probability column) are skipped for the multiclass case.
     - It may not fully represent the overall accuracy of the model if the cost of false positives and false negatives
     are extremely different, or if the dataset is heavily imbalanced.
     """
@@ -63,11 +73,10 @@ def PrecisionRecallCurve(
         raise SkipTestError("Skipping PrecisionRecallCurve for Foundation models")
 
     y_true = dataset.y
-    # Binary-only by design: multiclass is skipped, not handled (unlike MinimumF1Score).
-    if len(np.unique(y_true)) > 2:
-        raise SkipTestError(
-            "Precision Recall Curve is only supported for binary classification models"
-        )
+    classes = np.unique(y_true)
+
+    if len(classes) > 2:
+        return _multiclass_pr_curve(model, dataset)
 
     precision, recall, _ = precision_recall_curve(y_true, dataset.y_prob(model))
 
@@ -94,3 +103,80 @@ def PrecisionRecallCurve(
         model=model.input_id,
         dataset=dataset.input_id,
     )
+
+
+def _multiclass_pr_curve(
+    model: VMModel, dataset: VMDataset
+) -> Tuple[go.Figure, RawData]:
+    """One-vs-rest precision-recall curves for a multiclass model.
+
+    Needs the full per-class probability matrix, which the stored single
+    probability column cannot provide; the shared helper reaches the underlying
+    estimator, aligns the probability columns to the training class order and
+    skips models that cannot supply a matching matrix.
+    """
+    aligned = multiclass_proba(model, dataset, "Precision-Recall Curve")
+    y_bin = aligned.y_bin
+    y_prob = aligned.y_prob
+
+    traces = []
+    raw_precision = {}
+    raw_recall = {}
+    raw_ap = {}
+    palette = ["#DE257E", "#1F77B4", "#2CA02C", "#FF7F0E", "#9467BD", "#8C564B"]
+    for plot_i, (i, cls) in enumerate(
+        zip(aligned.present_indices, aligned.classes_present)
+    ):
+        precision, recall, _ = precision_recall_curve(y_bin[:, i], y_prob[:, i])
+        ap = average_precision_score(y_bin[:, i], y_prob[:, i])
+        key = str(cls)
+        raw_precision[key] = precision
+        raw_recall[key] = recall
+        raw_ap[key] = ap
+        traces.append(
+            go.Scatter(
+                x=recall,
+                y=precision,
+                mode="lines",
+                name=f"Class {key} (AP = {ap:.2f})",
+                line=dict(color=palette[plot_i % len(palette)]),
+            )
+        )
+
+    # Micro-average across the one-vs-rest decisions of the present classes.
+    present = aligned.present_indices
+    y_bin = y_bin[:, present]
+    y_prob = y_prob[:, present]
+    micro_precision, micro_recall, _ = precision_recall_curve(
+        y_bin.ravel(), y_prob.ravel()
+    )
+    micro_ap = average_precision_score(y_bin, y_prob, average="micro")
+    raw_precision["micro"] = micro_precision
+    raw_recall["micro"] = micro_recall
+    raw_ap["micro"] = micro_ap
+    traces.append(
+        go.Scatter(
+            x=micro_recall,
+            y=micro_precision,
+            mode="lines",
+            name=f"Micro-average (AP = {micro_ap:.2f})",
+            line=dict(color="black", dash="dot"),
+        )
+    )
+
+    fig = go.Figure(
+        data=traces,
+        layout=go.Layout(
+            title=f"Precision-Recall Curve (one-vs-rest) for {model.input_id} on {dataset.input_id}",
+            xaxis=dict(title="Recall"),
+            yaxis=dict(title="Precision"),
+        ),
+    )
+
+    return fig, RawData(
+        precision=raw_precision,
+        recall=raw_recall,
+        average_precision=raw_ap,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )