validmind 2.13.5__tar.gz → 2.13.7__tar.gz

{validmind-2.13.5 → validmind-2.13.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: validmind
-Version: 2.13.5
+Version: 2.13.7
 Summary: ValidMind Library
 Author-email: Andres Rodriguez <andres@validmind.ai>, Juan Martinez <juan@validmind.ai>, Anil Sorathiya <anil@validmind.ai>, Luis Pallares <luis@validmind.ai>, John Walz <john@validmind.ai>
 License:                          DUAL LICENSE NOTICE
@@ -726,7 +726,9 @@ Requires-Dist: langchain-google-genai; extra == 'all'
 Requires-Dist: langchain-openai>=0.1.8; extra == 'all'
 Requires-Dist: langchain<0.4.0,>=0.2.0; extra == 'all'
 Requires-Dist: langdetect; extra == 'all'
+Requires-Dist: llvmlite<0.48,>=0.47.0; (python_version >= '3.14') and extra == 'all'
 Requires-Dist: nltk<4.0.0,>=3.8.1; extra == 'all'
+Requires-Dist: numba>=0.65.1; (python_version >= '3.14') and extra == 'all'
 Requires-Dist: pyarrow<16; (python_version < '3.13') and extra == 'all'
 Requires-Dist: pyarrow<21,>=20; (python_version == '3.13') and extra == 'all'
 Requires-Dist: pyarrow<23,>=22; (python_version >= '3.14') and extra == 'all'
@@ -753,6 +755,8 @@ Requires-Dist: pyarrow<16; (python_version < '3.13') and extra == 'datasets'
 Requires-Dist: pyarrow<21,>=20; (python_version == '3.13') and extra == 'datasets'
 Requires-Dist: pyarrow<23,>=22; (python_version >= '3.14') and extra == 'datasets'
 Provides-Extra: explainability
+Requires-Dist: llvmlite<0.48,>=0.47.0; (python_version >= '3.14') and extra == 'explainability'
+Requires-Dist: numba>=0.65.1; (python_version >= '3.14') and extra == 'explainability'
 Requires-Dist: shap>=0.46.0; extra == 'explainability'
 Provides-Extra: huggingface
 Requires-Dist: sentencepiece<0.3.0,>=0.2.0; extra == 'huggingface'

{validmind-2.13.5 → validmind-2.13.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "validmind"
-version = "2.13.5"
+version = "2.13.7"
 description = "ValidMind Library"
 readme = "README.pypi.md"
 requires-python = ">=3.9,<3.15"
@@ -65,6 +65,8 @@ all = [
   "pyarrow (>=22,<23) ; python_version >= '3.14'",
   "arch (>=7.0.0)",
   "shap (>=0.46.0)",
+  "numba (>=0.65.1) ; python_version >= '3.14'",
+  "llvmlite (>=0.47.0,<0.48) ; python_version >= '3.14'",
   "scorecardpy==0.1.9.6",
   "setuptools (<82)",
 ]
@@ -107,7 +109,11 @@ nlp = [
 pytorch = ["torch (>=2.0.0)"]
 stats = ["scipy", "statsmodels (>=0.14.2,<0.15.0)", "arch (>=7.0.0)"]
 xgboost = ["xgboost (>=1.5.2,<3)"]
-explainability = ["shap (>=0.46.0)"]
+explainability = [
+  "shap (>=0.46.0)",
+  "numba (>=0.65.1) ; python_version >= '3.14'",
+  "llvmlite (>=0.47.0,<0.48) ; python_version >= '3.14'",
+]
 credit_risk = ["scorecardpy==0.1.9.6", "setuptools (<82)"]
 datasets = [
   "datasets (>=2.10.0,<3.0.0) ; python_version < '3.14'",

validmind-2.13.7/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.13.7"

{validmind-2.13.5 → validmind-2.13.7}/validmind/ai/utils.py

@@ -17,6 +17,7 @@ __client = None
 __model = None
 __judge_llm = None
 __judge_embeddings = None
+__judge_llm_explicitly_set = False
 OPENAI_MODEL = "gpt-4.1"
 OPENAI_EMBEDDINGS_MODEL = "text-embedding-3-small"
 GEMINI_MODEL = "gemini-2.5-pro"
@@ -238,14 +239,95 @@ def _unwrap_deepeval_response(response):
     return getattr(response, "content", response)
 
 
-def _build_gemini_deepeval_model(model):
+def _is_reason_only_schema(schema):
+    """True when ``schema`` is a single-field ``{reason}`` explanation schema.
+
+    DeepEval's ``*ScoreReason`` schemas carry only the human-readable explanation
+    and do not affect the metric score, so a missing value can be degraded safely.
+    """
+    fields = getattr(schema, "model_fields", None) or getattr(
+        schema, "__fields__", None
+    )
+    try:
+        return set(fields.keys()) == {"reason"}
+    except Exception:
+        return False
+
+
+def _require_structured_response(result, schema):
+    """Guard against a judge LLM returning no parseable structured output.
+
+    Gemini's default function-calling path yields ``None`` when the model
+    response contains no tool call. Returning that ``None`` to DeepEval surfaces
+    as a cryptic ``'NoneType' object has no attribute 'reason'``.
+
+    For reason-only schemas the metric score is already computed from earlier
+    verdict calls, so a missing explanation is cosmetic: degrade to a placeholder
+    reason and keep the score. Score-bearing schemas (statements, verdicts) are
+    never faked -- doing so would yield a silently wrong score -- so those raise.
+    """
+    if result is not None:
+        return result
+
+    schema_name = getattr(schema, "__name__", schema)
+
+    if _is_reason_only_schema(schema):
+        logger.warning(
+            "Judge LLM returned no structured output for %s; recording the score "
+            "with an empty explanation.",
+            schema_name,
+        )
+        return schema(reason="N/A (judge LLM returned no explanation)")
+
+    raise ValueError(
+        f"Judge LLM returned no parseable structured output for {schema_name}. "
+        "This usually means the model produced no tool call for the requested "
+        "schema. Try a different judge model, or check provider quotas and "
+        "safety settings."
+    )
+
+
+def _structured_generate(chat_model, prompt, schema):
+    """Invoke ``chat_model`` for structured output, retrying via json_mode.
+
+    Gemini's default function-calling path can return ``None`` when the model
+    emits no tool call; json_mode binds a response schema and is deterministic.
+    """
+    result = chat_model.with_structured_output(schema).invoke(prompt)
+    if result is None:
+        try:
+            result = chat_model.with_structured_output(
+                schema, method="json_mode"
+            ).invoke(prompt)
+        except Exception:
+            pass
+    return _require_structured_response(result, schema)
+
+
+async def _a_structured_generate(chat_model, prompt, schema):
+    """Async counterpart of :func:`_structured_generate`."""
+    result = await chat_model.with_structured_output(schema).ainvoke(prompt)
+    if result is None:
+        try:
+            result = await chat_model.with_structured_output(
+                schema, method="json_mode"
+            ).ainvoke(prompt)
+        except Exception:
+            pass
+    return _require_structured_response(result, schema)
+
+
+def _build_langchain_deepeval_model(chat_model):
+    """Wrap any LangChain BaseChatModel as a DeepEval-compatible model.
+
+    Used when set_judge_config() has been called so that DeepEval scorers
+    honour the same judge model as prompt-validation and RAGAS tests.
+    """
     DeepEvalBaseLLM = _import_deepeval_base_llm()
-    judge_llm, _ = _build_gemini_judge_config(model)
 
-    class GeminiDeepEvalModel(DeepEvalBaseLLM):
-        def __init__(self, chat_model, model_name):
-            self._chat_model = chat_model
-            self._model_name = model_name
+    class LangChainDeepEvalModel(DeepEvalBaseLLM):
+        def __init__(self, model):
+            self._chat_model = model
             self.model = self.load_model()
 
         def load_model(self, *args, **kwargs):
@@ -254,27 +336,28 @@ def _build_gemini_deepeval_model(model):
         def generate(self, prompt: str, schema=None):
             chat_model = self.load_model()
             if schema is not None and hasattr(chat_model, "with_structured_output"):
-                response = chat_model.with_structured_output(schema).invoke(prompt)
-            else:
-                response = chat_model.invoke(prompt)
-
-            return _unwrap_deepeval_response(response)
+                return _structured_generate(chat_model, prompt, schema)
+            return _unwrap_deepeval_response(chat_model.invoke(prompt))
 
         async def a_generate(self, prompt: str, schema=None):
             chat_model = self.load_model()
             if schema is not None and hasattr(chat_model, "with_structured_output"):
-                response = await chat_model.with_structured_output(schema).ainvoke(
-                    prompt
-                )
-            else:
-                response = await chat_model.ainvoke(prompt)
-
-            return _unwrap_deepeval_response(response)
+                return await _a_structured_generate(chat_model, prompt, schema)
+            return _unwrap_deepeval_response(await chat_model.ainvoke(prompt))
 
         def get_model_name(self, *args, **kwargs):
-            return self._model_name
+            for attr in ("model", "model_name", "azure_deployment"):
+                val = getattr(self._chat_model, attr, None)
+                if val:
+                    return val
+            return type(self._chat_model).__name__
+
+    return LangChainDeepEvalModel(chat_model)
+
 
-    return GeminiDeepEvalModel(judge_llm, model)
+def _build_gemini_deepeval_model(model):
+    judge_llm, _ = _build_gemini_judge_config(model)
+    return _build_langchain_deepeval_model(judge_llm)
 
 
 def run_deepeval_evaluation(*, test_cases, metrics):
@@ -332,6 +415,12 @@ def get_deepeval_model():
     OpenAI/Azure scorers currently pass a model string. Gemini support requires a
     native DeepEval model object so the provider can be configured correctly.
     """
+    if (
+        __judge_llm_explicitly_set
+        and __judge_llm is not None
+        and __judge_embeddings is not None
+    ):
+        return _build_langchain_deepeval_model(__judge_llm)
 
     provider = _get_configured_provider()
     _, model = get_client_and_model()
@@ -359,7 +448,7 @@ def get_deepeval_model():
 
 
 def set_judge_config(judge_llm, judge_embeddings):
-    global __judge_llm, __judge_embeddings
+    global __judge_llm, __judge_embeddings, __judge_llm_explicitly_set
     try:
         from langchain_core.embeddings import Embeddings
         from langchain_core.language_models.chat_models import BaseChatModel
@@ -372,12 +461,13 @@ def set_judge_config(judge_llm, judge_embeddings):
     ):
         __judge_llm = judge_llm
         __judge_embeddings = judge_embeddings
-        # Assuming 'your_object' is the object you want to check
+        __judge_llm_explicitly_set = True
     elif isinstance(judge_llm, FunctionModel) and isinstance(
         judge_embeddings, FunctionModel
     ):
         __judge_llm = judge_llm.model
         __judge_embeddings = judge_embeddings.model
+        __judge_llm_explicitly_set = True
     else:
         raise ValueError(
             "Provided Judge LLM/Embeddings are not Langchain compatible. Ensure the judge LLM/embedding provided are an instance of "

{validmind-2.13.5 → validmind-2.13.7}/validmind/tests/data_validation/IQROutliersBarPlot.py

@@ -5,6 +5,7 @@
 
 from typing import Tuple
 
+import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
@@ -76,9 +77,12 @@ def IQROutliersBarPlot(
     """
     df = dataset.df
 
-    # Exclude binary/boolean features (IQR is not meaningful and quantile fails on bool)
+    # Exclude boolean and binary features. The IQR is not meaningful for them and
+    # `quantile` raises "numpy boolean subtract" on boolean dtype columns.
     eligible_columns = [
-        col for col in dataset.feature_columns_numeric if len(df[col].unique()) > 2
+        col
+        for col in dataset.feature_columns_numeric
+        if not pd.api.types.is_bool_dtype(df[col]) and df[col].nunique() > 2
     ]
 
     figures = []

{validmind-2.13.5 → validmind-2.13.7}/validmind/tests/data_validation/IQROutliersTable.py

@@ -5,6 +5,8 @@
 
 from typing import Any, Dict, Tuple
 
+import pandas as pd
+
 from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset
 
@@ -72,8 +74,9 @@ def IQROutliersTable(
     all_outliers = {}
 
     for col in dataset.feature_columns_numeric:
-        # Skip binary features
-        if len(df[col].unique()) <= 2:
+        # Skip boolean and binary features. The IQR is not meaningful for them and
+        # `quantile` raises "numpy boolean subtract" on boolean dtype columns.
+        if pd.api.types.is_bool_dtype(df[col]) or df[col].nunique() <= 2:
             continue
 
         outliers = compute_outliers(df[col], threshold)

{validmind-2.13.5 → validmind-2.13.7}/validmind/tests/model_validation/ModelPredictionResiduals.py

@@ -12,7 +12,7 @@ from validmind.errors import MissingDependencyError
 from validmind.vm_models import VMDataset, VMModel
 
 try:
-    from scipy.stats import kstest
+    from scipy.stats import kstest, norm
 except ImportError as e:
     if "scipy" in str(e):
         raise MissingDependencyError(
@@ -109,7 +109,7 @@ def ModelPredictionResiduals(
 
     # Perform KS normality test
     ks_stat, p_value = kstest(
-        residuals, "norm", args=(residuals.mean(), residuals.std())
+        residuals, norm(loc=residuals.mean(), scale=residuals.std()).cdf
     )
     ks_normality = "Normal" if p_value > p_value_threshold else "Not Normal"
 

{validmind-2.13.5 → validmind-2.13.7}/validmind/tests/model_validation/sklearn/CalibrationCurve.py

@@ -4,10 +4,12 @@
 
 from typing import Tuple
 
+import numpy as np
 import plotly.graph_objects as go
 from sklearn.calibration import calibration_curve
 
 from validmind import tags, tasks
+from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 from validmind.vm_models.result import RawData
 
@@ -70,6 +72,14 @@ def CalibrationCurve(
     - Assumes bin boundaries are appropriate for the problem
     - May be affected by class imbalance
     """
+    # Binary-only by design: sklearn's calibration_curve raises a cryptic
+    # "pos_label is not specified" error on multiclass targets, so skip cleanly
+    # like ROCCurve/PrecisionRecallCurve rather than crashing.
+    if len(np.unique(dataset.y)) > 2:
+        raise SkipTestError(
+            "Calibration Curve is only supported for binary classification models"
+        )
+
     prob_true, prob_pred = calibration_curve(
         dataset.y, dataset.y_prob(model), n_bins=n_bins
     )

{validmind-2.13.5 → validmind-2.13.7}/validmind/tests/model_validation/sklearn/ConfusionMatrix.py

@@ -72,15 +72,20 @@ def ConfusionMatrix(
     - It mainly serves as a descriptive tool and does not offer the capability for statistical hypothesis testing.
     - Risks of misinterpretation exist because the matrix doesn't directly provide precision, recall, or F1-score data.
     These metrics have to be computed separately.
+    - The `threshold` parameter only applies to binary classification (it splits a single positive-class probability
+    into two classes). For multiclass targets the model's argmax class predictions are used and `threshold` is ignored.
     """
-    # Get predictions using threshold for binary classification if possible
-    if hasattr(model.model, "predict_proba"):
+    # The `threshold` only has a meaning for binary classification (it splits a
+    # single positive-class probability into two classes). For multiclass we use
+    # the model's argmax class predictions directly, since thresholding a
+    # single probability column would silently produce wrong labels.
+    n_classes = len(np.unique(dataset.y))
+    if n_classes == 2 and hasattr(model.model, "predict_proba"):
         y_prob = dataset.y_prob(model)
         # Handle both 1D and 2D probability arrays
         if y_prob.ndim == 2:
-            y_pred = (y_prob[:, 1] > threshold).astype(int)
-        else:
-            y_pred = (y_prob > threshold).astype(int)
+            y_prob = y_prob[:, 1]
+        y_pred = (y_prob > threshold).astype(int)
     else:
         y_pred = dataset.y_pred(model)
 

{validmind-2.13.5 → validmind-2.13.7}/validmind/tests/model_validation/sklearn/MinimumF1Score.py

@@ -58,10 +58,19 @@ def MinimumF1Score(
     closely with specific requirements.
     """
 
-    if len(np.unique(dataset.y)) > 2:
-        score = f1_score(dataset.y, dataset.y_pred(model), average="macro")
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Decide the averaging method from the labels f1_score actually sees -- the
+    # union of y_true and y_pred. Inspecting y_true alone is not enough: a split
+    # whose true labels collapse to <=2 classes while the model predicts a third
+    # class is still multiclass to sklearn, and average="binary" would raise
+    # "Target is multiclass but average='binary'".
+    n_classes = len(np.unique(np.concatenate([y_true, y_pred])))
+    if n_classes > 2:
+        score = f1_score(y_true, y_pred, average="macro")
     else:
-        score = f1_score(dataset.y, dataset.y_pred(model))
+        score = f1_score(y_true, y_pred)
 
     return (
         [

{validmind-2.13.5 → validmind-2.13.7}/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py

@@ -15,6 +15,8 @@ from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset, VMModel
 
+from ._diagnosis_metrics import bind_averaging, full_labels, multiclass_auc
+
 logger = get_logger(__name__)
 
 # TODO: A couple of improvements here could be to:
@@ -96,6 +98,10 @@ def _compute_metrics(
     feature_column: str,
     metric: str,
     is_classification: bool,
+    average: str = None,
+    pos_label=None,
+    labels: list = None,
+    is_multiclass: bool = False,
 ) -> None:
     results["slice"].append(str(region))
     results["shape"].append(df_region.shape[0])
@@ -115,11 +121,25 @@ def _compute_metrics(
         if len(np.unique(y_true)) == 1:
             return results[metric].append(0)
 
+        # The library retains only a single probability column, so a
+        # probability-based multiclass ROC AUC is not possible; fall back to the
+        # label-binarize convention used elsewhere for multiclass targets.
+        if is_multiclass:
+            return results[metric].append(
+                multiclass_auc(y_true, df_region[pred_column].values, labels)
+            )
+
         return results[metric].append(
             metric_func(y_true, df_region[prob_column].values)
         )
 
-    return results[metric].append(metric_func(y_true, df_region[pred_column].values))
+    # Bind averaging so precision/recall/F1 handle multiclass targets and binary
+    # targets encoded outside {0, 1}; accuracy and regression metrics pass through.
+    return results[metric].append(
+        bind_averaging(metric_func, average, pos_label)(
+            y_true, df_region[pred_column].values
+        )
+    )
 
 
 def _plot_overfit_regions(
@@ -253,6 +273,18 @@ def OverfitDiagnosis(
         train_df[prob_column] = datasets[0].y_prob(model)
         test_df[prob_column] = datasets[1].y_prob(model)
 
+    # Resolve the label space once so every feature slice is scored consistently:
+    # multiclass targets use macro averaging (and label-binarize AUC), binary
+    # targets keep the positive label so non-{0, 1} encodings don't break.
+    if is_classification:
+        labels = full_labels(datasets, model)
+        is_multiclass = len(labels) > 2
+        average, pos_label = (
+            ("macro", None) if is_multiclass else ("binary", labels[-1])
+        )
+    else:
+        labels, is_multiclass, average, pos_label = None, False, None, None
+
     test_results = []
     figures = []
     results_headers = ["slice", "shape", "feature", metric]
@@ -277,6 +309,10 @@ def OverfitDiagnosis(
                 pred_column=pred_column,
                 metric=metric,
                 is_classification=is_classification,
+                average=average,
+                pos_label=pos_label,
+                labels=labels,
+                is_multiclass=is_multiclass,
             )
             df_test_region = test_df[
                 (test_df[feature_column] > region.left)
@@ -292,6 +328,10 @@ def OverfitDiagnosis(
                 pred_column=pred_column,
                 metric=metric,
                 is_classification=is_classification,
+                average=average,
+                pos_label=pos_label,
+                labels=labels,
+                is_multiclass=is_multiclass,
             )
 
         results = _prepare_results(results_train, results_test, metric)