validmind 2.7.12__py3-none-any.whl → 2.8.12__py3-none-any.whl

validmind/tests/ongoing_monitoring/ROCCurveDrift.py

@@ -8,7 +8,7 @@ import numpy as np
 import plotly.graph_objects as go
 from sklearn.metrics import roc_auc_score, roc_curve
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
@@ -147,4 +147,18 @@ def ROCCurveDrift(datasets: List[VMDataset], model: VMModel):
         height=500,
     )
 
-    return fig1, fig2
+    return (
+        fig1,
+        fig2,
+        RawData(
+            fpr_ref=fpr_ref,
+            tpr_ref=tpr_ref,
+            auc_ref=auc_ref,
+            fpr_mon=fpr_mon,
+            tpr_mon=tpr_mon,
+            auc_mon=auc_mon,
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
+    )

validmind/tests/ongoing_monitoring/ScoreBandsDrift.py

@@ -7,7 +7,7 @@ from typing import List
 import numpy as np
 import pandas as pd
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -209,4 +209,13 @@ def ScoreBandsDrift(
         tables[table_name] = pd.DataFrame(rows)
         all_passed &= metric_passed
 
-    return tables, all_passed
+    # Collect raw data
+    raw_data = RawData(
+        ref_results=ref_results,
+        mon_results=mon_results,
+        model=model.input_id,
+        dataset_reference=datasets[0].input_id,
+        dataset_monitoring=datasets[1].input_id,
+    )
+
+    return tables, all_passed, raw_data

validmind/tests/ongoing_monitoring/TargetPredictionDistributionPlot.py

@@ -7,7 +7,7 @@ import plotly.figure_factory as ff
 import plotly.graph_objects as go
 from scipy.stats import kurtosis, skew
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tags("visualization")
@@ -142,4 +142,15 @@ def TargetPredictionDistributionPlot(datasets, model, drift_pct_threshold=20):
 
     pass_fail_bool = (moments["Pass/Fail"] == "Pass").all()
 
-    return ({"Distribution Moments": moments}, fig, pass_fail_bool)
+    return (
+        {"Distribution Moments": moments},
+        fig,
+        pass_fail_bool,
+        RawData(
+            pred_ref=pred_ref,
+            pred_monitor=pred_monitor,
+            model=model.input_id,
+            dataset_reference=datasets[0].input_id,
+            dataset_monitoring=datasets[1].input_id,
+        ),
+    )

validmind/tests/prompt_validation/Clarity.py

@@ -106,5 +106,5 @@ def Clarity(model, min_threshold=7):
             }
         ],
         passed,
-        RawData(response=response),
+        RawData(response=response, model=model.input_id),
     )

validmind/tests/prompt_validation/NegativeInstruction.py

@@ -117,5 +117,5 @@ def NegativeInstruction(model, min_threshold=7):
             }
         ],
         passed,
-        RawData(model_response=response),
+        RawData(model_response=response, model=model.input_id),
     )

validmind/tests/prompt_validation/Robustness.py

@@ -130,5 +130,10 @@ def Robustness(model, dataset, num_tests=10):
     return (
         results,
         all(result["Pass/Fail"] == "Pass" for result in results),
-        RawData(generated_inputs=generated_inputs, responses=responses),
+        RawData(
+            generated_inputs=generated_inputs,
+            responses=responses,
+            model=model.input_id,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/prompt_validation/Specificity.py

@@ -113,5 +113,5 @@ def Specificity(model, min_threshold=7):
             }
         ],
         passed,
-        RawData(response=response),
+        RawData(response=response, model=model.input_id),
     )

validmind/tests/prompt_validation/ai_powered_test.py

@@ -4,8 +4,7 @@
 
 import re
 
-from validmind.ai.utils import get_client_and_model
-from validmind.client_config import client_config
+from validmind.ai.utils import get_client_and_model, is_configured
 
 missing_prompt_message = """
 Cannot run prompt validation tests on a model with no prompt.
@@ -25,9 +24,11 @@ def call_model(
     system_prompt: str, user_prompt: str, temperature: float = 0.0, seed: int = 42
 ):
     """Call LLM with the given prompts and return the response"""
-    if not client_config.can_generate_llm_test_descriptions():
+    if not is_configured():
         raise ValueError(
-            "LLM based descriptions are not enabled for your organization."
+            "LLM is not configured. Please set an `OPENAI_API_KEY` environment variable "
+            "or ensure that you are connected to the ValidMind API and ValidMind AI is "
+            "enabled for your account."
         )
 
     client, model = get_client_and_model()

validmind/tests/run.py

@@ -256,7 +256,7 @@ def _run_comparison_test(
     combined_outputs, combined_inputs, combined_params = combine_results(results)
 
     return build_test_result(
-        outputs=tuple(combined_outputs),
+        outputs=combined_outputs,
         test_id=test_id,
         test_doc=test_doc,
         inputs=combined_inputs,
@@ -400,5 +400,9 @@ def run_test(  # noqa: C901
 
 
 def print_env():
+    """Prints a log of the running environment for debugging.
+
+    Output includes: ValidMind Library version, operating system details, installed dependencies, and the ISO 8601 timestamp at log creation.
+    """
     e = _get_run_metadata()
     pprint.pp(e)

validmind/utils.py

@@ -60,6 +60,19 @@ pylab.rcParams.update(params)
 logger = get_logger(__name__)
 
 
+def parse_version(version: str) -> tuple[int, ...]:
+    """
+    Parse a semver version string into a tuple of major, minor, patch integers
+
+    Args:
+        version (str): The semantic version string to parse
+
+    Returns:
+        tuple[int, ...]: A tuple of major, minor, patch integers
+    """
+    return tuple(int(x) for x in version.split(".")[:3])
+
+
 def is_notebook() -> bool:
     """
     Checks if the code is running in a Jupyter notebook or IPython shell

validmind/vm_models/result/result.py

@@ -171,6 +171,7 @@ class TestResult(Result):
     metadata: Optional[Dict[str, Any]] = None
     _was_description_generated: bool = False
     _unsafe: bool = False
+    _client_config_cache: Optional[Any] = None
 
     def __post_init__(self):
         if self.ref_id is None:
@@ -329,13 +330,50 @@ class TestResult(Result):
 
         return VBox(widgets)
 
+    @classmethod
+    def _get_client_config(cls):
+        """Get the client config, loading it if not cached"""
+        if cls._client_config_cache is None:
+            api_client.reload()
+            cls._client_config_cache = api_client.client_config
+
+            if cls._client_config_cache is None:
+                raise ValueError(
+                    "Failed to load client config: api_client.client_config is None"
+                )
+
+            if not hasattr(cls._client_config_cache, "documentation_template"):
+                raise ValueError(
+                    "Invalid client config: missing documentation_template"
+                )
+
+        return cls._client_config_cache
+
+    def check_result_id_exist(self):
+        """Check if the result_id exists in any test block across all sections"""
+        client_config = self._get_client_config()
+
+        # Iterate through all sections
+        for section in client_config.documentation_template["sections"]:
+            blocks = section.get("contents", [])
+            # Check each block in the section
+            for block in blocks:
+                if (
+                    block.get("content_type") == "test"
+                    and block.get("content_id") == self.result_id
+                ):
+                    return
+
+        logger.info(
+            f"Test driven block with result_id {self.result_id} does not exist in model's document"
+        )
+
     def _validate_section_id_for_block(
         self, section_id: str, position: Union[int, None] = None
     ):
         """Validate the section_id exits on the template before logging"""
-        api_client.reload()
+        client_config = self._get_client_config()
         found = False
-        client_config = api_client.client_config
 
         for section in client_config.documentation_template["sections"]:
             if section["id"] == section_id:
@@ -440,6 +478,9 @@ class TestResult(Result):
             unsafe (bool): If True, log the result even if it contains sensitive data
                 i.e. raw data from input datasets
         """
+
+        self.check_result_id_exist()
+
         if not unsafe:
             for table in self.tables or []:
                 check_for_sensitive_data(table.data, self._get_flat_inputs())

{validmind-2.7.12.dist-info → validmind-2.8.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: validmind
-Version: 2.7.12
+Version: 2.8.12
 Summary: ValidMind Library
 License: Commercial License
 Author: Andres Rodriguez
@@ -131,6 +131,7 @@ The ValidMind Library has optional dependencies that can be installed separately
 - **R Models**: To use R models with the ValidMind Library, install the `r` extra:
 
     ```bash
-    pip install validmind[r-support]
+    pip install validmind
+    pip install rpy2
     ```