validmind 2.13.5__tar.gz → 2.13.6__tar.gz

{validmind-2.13.5 → validmind-2.13.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: validmind
-Version: 2.13.5
+Version: 2.13.6
 Summary: ValidMind Library
 Author-email: Andres Rodriguez <andres@validmind.ai>, Juan Martinez <juan@validmind.ai>, Anil Sorathiya <anil@validmind.ai>, Luis Pallares <luis@validmind.ai>, John Walz <john@validmind.ai>
 License:                          DUAL LICENSE NOTICE
@@ -726,7 +726,9 @@ Requires-Dist: langchain-google-genai; extra == 'all'
 Requires-Dist: langchain-openai>=0.1.8; extra == 'all'
 Requires-Dist: langchain<0.4.0,>=0.2.0; extra == 'all'
 Requires-Dist: langdetect; extra == 'all'
+Requires-Dist: llvmlite<0.48,>=0.47.0; (python_version >= '3.14') and extra == 'all'
 Requires-Dist: nltk<4.0.0,>=3.8.1; extra == 'all'
+Requires-Dist: numba>=0.65.1; (python_version >= '3.14') and extra == 'all'
 Requires-Dist: pyarrow<16; (python_version < '3.13') and extra == 'all'
 Requires-Dist: pyarrow<21,>=20; (python_version == '3.13') and extra == 'all'
 Requires-Dist: pyarrow<23,>=22; (python_version >= '3.14') and extra == 'all'
@@ -753,6 +755,8 @@ Requires-Dist: pyarrow<16; (python_version < '3.13') and extra == 'datasets'
 Requires-Dist: pyarrow<21,>=20; (python_version == '3.13') and extra == 'datasets'
 Requires-Dist: pyarrow<23,>=22; (python_version >= '3.14') and extra == 'datasets'
 Provides-Extra: explainability
+Requires-Dist: llvmlite<0.48,>=0.47.0; (python_version >= '3.14') and extra == 'explainability'
+Requires-Dist: numba>=0.65.1; (python_version >= '3.14') and extra == 'explainability'
 Requires-Dist: shap>=0.46.0; extra == 'explainability'
 Provides-Extra: huggingface
 Requires-Dist: sentencepiece<0.3.0,>=0.2.0; extra == 'huggingface'

{validmind-2.13.5 → validmind-2.13.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "validmind"
-version = "2.13.5"
+version = "2.13.6"
 description = "ValidMind Library"
 readme = "README.pypi.md"
 requires-python = ">=3.9,<3.15"
@@ -65,6 +65,8 @@ all = [
   "pyarrow (>=22,<23) ; python_version >= '3.14'",
   "arch (>=7.0.0)",
   "shap (>=0.46.0)",
+  "numba (>=0.65.1) ; python_version >= '3.14'",
+  "llvmlite (>=0.47.0,<0.48) ; python_version >= '3.14'",
   "scorecardpy==0.1.9.6",
   "setuptools (<82)",
 ]
@@ -107,7 +109,11 @@ nlp = [
 pytorch = ["torch (>=2.0.0)"]
 stats = ["scipy", "statsmodels (>=0.14.2,<0.15.0)", "arch (>=7.0.0)"]
 xgboost = ["xgboost (>=1.5.2,<3)"]
-explainability = ["shap (>=0.46.0)"]
+explainability = [
+  "shap (>=0.46.0)",
+  "numba (>=0.65.1) ; python_version >= '3.14'",
+  "llvmlite (>=0.47.0,<0.48) ; python_version >= '3.14'",
+]
 credit_risk = ["scorecardpy==0.1.9.6", "setuptools (<82)"]
 datasets = [
   "datasets (>=2.10.0,<3.0.0) ; python_version < '3.14'",

validmind-2.13.6/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.13.6"

{validmind-2.13.5 → validmind-2.13.6}/validmind/ai/utils.py

@@ -17,6 +17,7 @@ __client = None
 __model = None
 __judge_llm = None
 __judge_embeddings = None
+__judge_llm_explicitly_set = False
 OPENAI_MODEL = "gpt-4.1"
 OPENAI_EMBEDDINGS_MODEL = "text-embedding-3-small"
 GEMINI_MODEL = "gemini-2.5-pro"
@@ -238,14 +239,95 @@ def _unwrap_deepeval_response(response):
     return getattr(response, "content", response)
 
 
-def _build_gemini_deepeval_model(model):
+def _is_reason_only_schema(schema):
+    """True when ``schema`` is a single-field ``{reason}`` explanation schema.
+
+    DeepEval's ``*ScoreReason`` schemas carry only the human-readable explanation
+    and do not affect the metric score, so a missing value can be degraded safely.
+    """
+    fields = getattr(schema, "model_fields", None) or getattr(
+        schema, "__fields__", None
+    )
+    try:
+        return set(fields.keys()) == {"reason"}
+    except Exception:
+        return False
+
+
+def _require_structured_response(result, schema):
+    """Guard against a judge LLM returning no parseable structured output.
+
+    Gemini's default function-calling path yields ``None`` when the model
+    response contains no tool call. Returning that ``None`` to DeepEval surfaces
+    as a cryptic ``'NoneType' object has no attribute 'reason'``.
+
+    For reason-only schemas the metric score is already computed from earlier
+    verdict calls, so a missing explanation is cosmetic: degrade to a placeholder
+    reason and keep the score. Score-bearing schemas (statements, verdicts) are
+    never faked -- doing so would yield a silently wrong score -- so those raise.
+    """
+    if result is not None:
+        return result
+
+    schema_name = getattr(schema, "__name__", schema)
+
+    if _is_reason_only_schema(schema):
+        logger.warning(
+            "Judge LLM returned no structured output for %s; recording the score "
+            "with an empty explanation.",
+            schema_name,
+        )
+        return schema(reason="N/A (judge LLM returned no explanation)")
+
+    raise ValueError(
+        f"Judge LLM returned no parseable structured output for {schema_name}. "
+        "This usually means the model produced no tool call for the requested "
+        "schema. Try a different judge model, or check provider quotas and "
+        "safety settings."
+    )
+
+
+def _structured_generate(chat_model, prompt, schema):
+    """Invoke ``chat_model`` for structured output, retrying via json_mode.
+
+    Gemini's default function-calling path can return ``None`` when the model
+    emits no tool call; json_mode binds a response schema and is deterministic.
+    """
+    result = chat_model.with_structured_output(schema).invoke(prompt)
+    if result is None:
+        try:
+            result = chat_model.with_structured_output(
+                schema, method="json_mode"
+            ).invoke(prompt)
+        except Exception:
+            pass
+    return _require_structured_response(result, schema)
+
+
+async def _a_structured_generate(chat_model, prompt, schema):
+    """Async counterpart of :func:`_structured_generate`."""
+    result = await chat_model.with_structured_output(schema).ainvoke(prompt)
+    if result is None:
+        try:
+            result = await chat_model.with_structured_output(
+                schema, method="json_mode"
+            ).ainvoke(prompt)
+        except Exception:
+            pass
+    return _require_structured_response(result, schema)
+
+
+def _build_langchain_deepeval_model(chat_model):
+    """Wrap any LangChain BaseChatModel as a DeepEval-compatible model.
+
+    Used when set_judge_config() has been called so that DeepEval scorers
+    honour the same judge model as prompt-validation and RAGAS tests.
+    """
     DeepEvalBaseLLM = _import_deepeval_base_llm()
-    judge_llm, _ = _build_gemini_judge_config(model)
 
-    class GeminiDeepEvalModel(DeepEvalBaseLLM):
-        def __init__(self, chat_model, model_name):
-            self._chat_model = chat_model
-            self._model_name = model_name
+    class LangChainDeepEvalModel(DeepEvalBaseLLM):
+        def __init__(self, model):
+            self._chat_model = model
             self.model = self.load_model()
 
         def load_model(self, *args, **kwargs):
@@ -254,27 +336,28 @@ def _build_gemini_deepeval_model(model):
         def generate(self, prompt: str, schema=None):
             chat_model = self.load_model()
             if schema is not None and hasattr(chat_model, "with_structured_output"):
-                response = chat_model.with_structured_output(schema).invoke(prompt)
-            else:
-                response = chat_model.invoke(prompt)
-
-            return _unwrap_deepeval_response(response)
+                return _structured_generate(chat_model, prompt, schema)
+            return _unwrap_deepeval_response(chat_model.invoke(prompt))
 
         async def a_generate(self, prompt: str, schema=None):
             chat_model = self.load_model()
             if schema is not None and hasattr(chat_model, "with_structured_output"):
-                response = await chat_model.with_structured_output(schema).ainvoke(
-                    prompt
-                )
-            else:
-                response = await chat_model.ainvoke(prompt)
-
-            return _unwrap_deepeval_response(response)
+                return await _a_structured_generate(chat_model, prompt, schema)
+            return _unwrap_deepeval_response(await chat_model.ainvoke(prompt))
 
         def get_model_name(self, *args, **kwargs):
-            return self._model_name
+            for attr in ("model", "model_name", "azure_deployment"):
+                val = getattr(self._chat_model, attr, None)
+                if val:
+                    return val
+            return type(self._chat_model).__name__
+
+    return LangChainDeepEvalModel(chat_model)
+
 
-    return GeminiDeepEvalModel(judge_llm, model)
+def _build_gemini_deepeval_model(model):
+    judge_llm, _ = _build_gemini_judge_config(model)
+    return _build_langchain_deepeval_model(judge_llm)
 
 
 def run_deepeval_evaluation(*, test_cases, metrics):
@@ -332,6 +415,12 @@ def get_deepeval_model():
     OpenAI/Azure scorers currently pass a model string. Gemini support requires a
     native DeepEval model object so the provider can be configured correctly.
     """
+    if (
+        __judge_llm_explicitly_set
+        and __judge_llm is not None
+        and __judge_embeddings is not None
+    ):
+        return _build_langchain_deepeval_model(__judge_llm)
 
     provider = _get_configured_provider()
     _, model = get_client_and_model()
@@ -359,7 +448,7 @@ def get_deepeval_model():
 
 
 def set_judge_config(judge_llm, judge_embeddings):
-    global __judge_llm, __judge_embeddings
+    global __judge_llm, __judge_embeddings, __judge_llm_explicitly_set
     try:
         from langchain_core.embeddings import Embeddings
         from langchain_core.language_models.chat_models import BaseChatModel
@@ -372,12 +461,13 @@ def set_judge_config(judge_llm, judge_embeddings):
     ):
         __judge_llm = judge_llm
         __judge_embeddings = judge_embeddings
-        # Assuming 'your_object' is the object you want to check
+        __judge_llm_explicitly_set = True
     elif isinstance(judge_llm, FunctionModel) and isinstance(
         judge_embeddings, FunctionModel
     ):
         __judge_llm = judge_llm.model
         __judge_embeddings = judge_embeddings.model
+        __judge_llm_explicitly_set = True
     else:
         raise ValueError(
             "Provided Judge LLM/Embeddings are not Langchain compatible. Ensure the judge LLM/embedding provided are an instance of "

{validmind-2.13.5 → validmind-2.13.6}/validmind/tests/data_validation/IQROutliersBarPlot.py

@@ -5,6 +5,7 @@
 
 from typing import Tuple
 
+import pandas as pd
 import plotly.graph_objects as go
 
 from validmind import RawData, tags, tasks
@@ -76,9 +77,12 @@ def IQROutliersBarPlot(
     """
     df = dataset.df
 
-    # Exclude binary/boolean features (IQR is not meaningful and quantile fails on bool)
+    # Exclude boolean and binary features. The IQR is not meaningful for them and
+    # `quantile` raises "numpy boolean subtract" on boolean dtype columns.
     eligible_columns = [
-        col for col in dataset.feature_columns_numeric if len(df[col].unique()) > 2
+        col
+        for col in dataset.feature_columns_numeric
+        if not pd.api.types.is_bool_dtype(df[col]) and df[col].nunique() > 2
     ]
 
     figures = []

{validmind-2.13.5 → validmind-2.13.6}/validmind/tests/data_validation/IQROutliersTable.py

@@ -5,6 +5,8 @@
 
 from typing import Any, Dict, Tuple
 
+import pandas as pd
+
 from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset
 
@@ -72,8 +74,9 @@ def IQROutliersTable(
     all_outliers = {}
 
     for col in dataset.feature_columns_numeric:
-        # Skip binary features
-        if len(df[col].unique()) <= 2:
+        # Skip boolean and binary features. The IQR is not meaningful for them and
+        # `quantile` raises "numpy boolean subtract" on boolean dtype columns.
+        if pd.api.types.is_bool_dtype(df[col]) or df[col].nunique() <= 2:
             continue
 
         outliers = compute_outliers(df[col], threshold)

{validmind-2.13.5 → validmind-2.13.6}/validmind/tests/model_validation/ModelPredictionResiduals.py

@@ -12,7 +12,7 @@ from validmind.errors import MissingDependencyError
 from validmind.vm_models import VMDataset, VMModel
 
 try:
-    from scipy.stats import kstest
+    from scipy.stats import kstest, norm
 except ImportError as e:
     if "scipy" in str(e):
         raise MissingDependencyError(
@@ -109,7 +109,7 @@ def ModelPredictionResiduals(
 
     # Perform KS normality test
     ks_stat, p_value = kstest(
-        residuals, "norm", args=(residuals.mean(), residuals.std())
+        residuals, norm(loc=residuals.mean(), scale=residuals.std()).cdf
     )
     ks_normality = "Normal" if p_value > p_value_threshold else "Not Normal"
 

{validmind-2.13.5 → validmind-2.13.6}/validmind/tests/model_validation/sklearn/MinimumF1Score.py

@@ -58,10 +58,19 @@ def MinimumF1Score(
     closely with specific requirements.
     """
 
-    if len(np.unique(dataset.y)) > 2:
-        score = f1_score(dataset.y, dataset.y_pred(model), average="macro")
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Decide the averaging method from the labels f1_score actually sees -- the
+    # union of y_true and y_pred. Inspecting y_true alone is not enough: a split
+    # whose true labels collapse to <=2 classes while the model predicts a third
+    # class is still multiclass to sklearn, and average="binary" would raise
+    # "Target is multiclass but average='binary'".
+    n_classes = len(np.unique(np.concatenate([y_true, y_pred])))
+    if n_classes > 2:
+        score = f1_score(y_true, y_pred, average="macro")
     else:
-        score = f1_score(dataset.y, dataset.y_pred(model))
+        score = f1_score(y_true, y_pred)
 
     return (
         [

{validmind-2.13.5 → validmind-2.13.6}/validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py

@@ -63,6 +63,7 @@ def PrecisionRecallCurve(
         raise SkipTestError("Skipping PrecisionRecallCurve for Foundation models")
 
     y_true = dataset.y
+    # Binary-only by design: multiclass is skipped, not handled (unlike MinimumF1Score).
     if len(np.unique(y_true)) > 2:
         raise SkipTestError(
             "Precision Recall Curve is only supported for binary classification models"

{validmind-2.13.5 → validmind-2.13.6}/validmind/tests/model_validation/sklearn/ROCCurve.py

@@ -68,6 +68,7 @@ def ROCCurve(model: VMModel, dataset: VMDataset) -> Tuple[go.Figure, RawData]:
     incorrect, provided that the model's ranking format is retained. This phenomenon is commonly termed the "Class
     Imbalance Problem".
     """
+    # Binary-only by design: multiclass is skipped, not handled (unlike MinimumF1Score).
     if len(np.unique(dataset.y)) > 2:
         raise SkipTestError(
             "ROC Curve is only supported for binary classification models"

{validmind-2.13.5 → validmind-2.13.6}/validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py

@@ -4,9 +4,9 @@
 
 from typing import Tuple
 
+import matplotlib
 import numpy as np
 import plotly.graph_objects as go
-from matplotlib import cm
 
 from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
@@ -75,7 +75,7 @@ def _plot_cumulative_prob(df, target_col, title):
 
     # Generate a colormap and convert to Plotly-accepted color format
     # Adjust 'viridis' to any other matplotlib colormap if desired
-    colormap = cm.get_cmap("viridis")
+    colormap = matplotlib.colormaps["viridis"]
 
     fig = go.Figure()
 

{validmind-2.13.5 → validmind-2.13.6}/validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py

@@ -4,8 +4,8 @@
 
 from typing import Tuple
 
+import matplotlib
 import plotly.graph_objects as go
-from matplotlib import cm
 
 from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
@@ -75,7 +75,7 @@ def _plot_prob_histogram(df, target_col, title):
 
     # Generate a colormap and convert to Plotly-accepted color format
     # Adjust 'viridis' to any other matplotlib colormap if desired
-    colormap = cm.get_cmap("viridis")
+    colormap = matplotlib.colormaps["viridis"]
 
     fig = go.Figure()
 

{validmind-2.13.5 → validmind-2.13.6}/validmind/tests/model_validation/statsmodels/ScorecardHistogram.py

@@ -2,8 +2,8 @@
 # Refer to the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import matplotlib
 import plotly.graph_objects as go
-from matplotlib import cm
 
 from validmind import tags, tasks
 from validmind.vm_models import VMDataset
@@ -77,7 +77,7 @@ def ScorecardHistogram(
 def _plot_score_histogram(df, score_col, target_col, title):
     # Generate a colormap and convert to Plotly-accepted color format
     # Adjust 'viridis' to any other matplotlib colormap if desired
-    colormap = cm.get_cmap("viridis")
+    colormap = matplotlib.colormaps["viridis"]
 
     fig = go.Figure()
 

validmind-2.13.5/validmind/__version__.py

@@ -1 +0,0 @@
-__version__ = "2.13.5"