validmind 2.2.5__py3-none-any.whl → 2.3.1__py3-none-any.whl

validmind/vm_models/dataset/dataset.py

@@ -195,7 +195,19 @@ class VMDataset:
         probability_column: str = None,
         probability_values: list = None,
         prediction_probabilities: list = None,  # DEPRECATED: use probability_values
+        **kwargs,
     ):
+        """Assign predictions and probabilities to the dataset.
+
+        Args:
+            model (VMModel): The model used to generate the predictions.
+            prediction_column (str, optional): The name of the column containing the predictions. Defaults to None.
+            prediction_values (list, optional): The values of the predictions. Defaults to None.
+            probability_column (str, optional): The name of the column containing the probabilities. Defaults to None.
+            probability_values (list, optional): The values of the probabilities. Defaults to None.
+            prediction_probabilities (list, optional): DEPRECATED: The values of the probabilities. Defaults to None.
+            kwargs: Additional keyword arguments that will get passed through to the model's `predict` method.
+        """
         if prediction_probabilities is not None:
             warnings.warn(
                 "The `prediction_probabilities` argument is deprecated. Use `probability_values` instead.",
@@ -226,7 +238,9 @@ class VMDataset:
 
         if prediction_values is None:
             X = self.df if isinstance(model, (FunctionModel, PipelineModel)) else self.x
-            probability_values, prediction_values = compute_predictions(model, X)
+            probability_values, prediction_values = compute_predictions(
+                model, X, **kwargs
+            )
 
         prediction_column = prediction_column or f"{model.input_id}_prediction"
         self._add_column(prediction_column, prediction_values)
@@ -356,8 +370,8 @@ class VMDataset:
         return as_df(self.df[self.probability_column(model)])
 
     def target_classes(self):
-        """Returns the unique number of target classes for the target (Y) variable"""
-        return [str(i) for i in np.unique(self.y)]
+        """Returns the target class labels or unique values of the target column."""
+        return self.target_class_labels or [str(i) for i in np.unique(self.y)]
 
     def __str__(self):
         return (

validmind/vm_models/dataset/utils.py

@@ -94,7 +94,7 @@ def _is_probabilties(output):
     return np.all((output >= 0) & (output <= 1)) and np.any((output > 0) & (output < 1))
 
 
-def compute_predictions(model, X) -> tuple:
+def compute_predictions(model, X, **kwargs) -> tuple:
     probability_values = None
 
     try:
@@ -108,7 +108,7 @@ def compute_predictions(model, X) -> tuple:
 
     try:
         logger.info("Running predict()... This may take a while")
-        prediction_values = model.predict(X)
+        prediction_values = model.predict(X, **kwargs)
         logger.info("Done running predict()")
     except MissingOrInvalidModelPredictFnError:
         raise MissingOrInvalidModelPredictFnError(

validmind/vm_models/model.py

@@ -114,7 +114,7 @@ class VMModel(ABC):
 
         self.__post_init__()
 
-    def __post_init__(self):
+    def __post_init__(self):  # noqa: B027
         """Allows child classes to add their own post-init logic"""
         pass
 

validmind/vm_models/test/metric.py

@@ -12,8 +12,8 @@ from typing import ClassVar, List, Optional, Union
 
 import pandas as pd
 
+from ...ai.test_descriptions import get_description_metadata
 from ...errors import MissingCacheResultsArgumentsError
-from ...utils import get_description_metadata
 from ..figure import Figure
 from .metric_result import MetricResult
 from .result_wrapper import MetricResultWrapper
@@ -36,13 +36,6 @@ class Metric(Test):
     # Instance Variables
     result: MetricResultWrapper = None  # populated by cache_results() method
 
-    @property
-    def key(self):
-        """
-        Keep the key for compatibility reasons
-        """
-        return self._key if hasattr(self, "_key") else self.name
-
     @abstractmethod
     def summary(self, metric_value: Optional[Union[dict, list, pd.DataFrame]] = None):
         """

validmind/vm_models/test/result_wrapper.py

@@ -7,7 +7,6 @@ Result Wrappers for test and metric results
 """
 import asyncio
 import json
-import os
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import Dict, List, Optional, Union
@@ -16,10 +15,10 @@ import pandas as pd
 from ipywidgets import HTML, GridBox, Layout, VBox
 
 from ... import api_client
-from ...ai import DescriptionFuture
+from ...ai.test_descriptions import AI_REVISION_NAME, DescriptionFuture
 from ...input_registry import input_registry
 from ...logging import get_logger
-from ...utils import NumpyEncoder, display, md_to_html, run_async, test_id_to_name
+from ...utils import NumpyEncoder, display, run_async, test_id_to_name
 from ..dataset import VMDataset
 from ..figure import Figure
 from .metric_result import MetricResult
@@ -31,31 +30,35 @@ logger = get_logger(__name__)
 
 
 async def update_metadata(content_id: str, text: str, _json: Union[Dict, List] = None):
-    """
-    Update the metadata of a content item. By default we don't
-    override the existing metadata, but we can override it by
-    setting the VM_OVERRIDE_METADATA environment variable to True
-    """
-    should_update = False
-
-    # check if the env variable is set to force overwriting metadata
-    if os.environ.get("VM_OVERRIDE_METADATA", "false").lower() == "true":
-        should_update = True
+    """Create or Update a Metadata Object"""
+    parts = content_id.split("::")
+    content_id = parts[0]
+    revision_name = parts[1] if len(parts) > 1 else None
 
-    # if not set, check if the content_id is a composite metric def
-    if not should_update and content_id.startswith("composite_metric_def:"):
-        # we always want composite metric definitions to be updated
-        should_update = True
+    # we always want composite metric definitions to be updated
+    should_update = content_id.startswith("composite_metric_def:")
 
-    # if not set, lets check if the metadata already exists
-    if not should_update:
+    # if we are updating a metric or test description, we check if the text
+    # has changed from the last time it was logged, and only update if it has
+    if content_id.split(":", 1)[0] in ["metric_description", "test_description"]:
         try:
-            await api_client.get_metadata(content_id)
-        except Exception:  # TODO: this shouldn't be a catch-all
-            # if the metadata doesn't exist, we should create (update) it
+            md = await api_client.get_metadata(content_id)
+            # if there is an existing description, only update it if the new one
+            # is different and is an AI-generated description
+            should_update = (
+                md["text"] != text if revision_name == AI_REVISION_NAME else False
+            )
+            logger.debug(f"Check if description has changed: {should_update}")
+        except Exception:
+            # if exception, assume its not created yet TODO: don't catch all
             should_update = True
 
     if should_update:
+        if revision_name:
+            content_id = f"{content_id}::{revision_name}"
+
+        logger.debug(f"Updating metadata for `{content_id}`")
+
         await api_client.log_metadata(content_id, text, _json)
 
 
@@ -102,12 +105,6 @@ class ResultWrapper(ABC):
 
         return self.to_widget()
 
-    def _markdown_description_to_html(self, description: str):
-        """
-        Convert a markdown string to html
-        """
-        return md_to_html(description)
-
     def _summary_tables_to_widget(self, summary: ResultSummary):
         """
         Create an ipywdiget representation of the summary tables
@@ -277,9 +274,7 @@ class MetricResultWrapper(ResultWrapper):
                 metric_description = metric_description.get_description()
                 self.result_metadata[0]["text"] = metric_description
 
-            vbox_children.append(
-                HTML(value=self._markdown_description_to_html(metric_description))
-            )
+            vbox_children.append(HTML(value=metric_description))
 
         if self.metric:
             if self.output_template:
@@ -464,9 +459,7 @@ class ThresholdTestResultWrapper(ResultWrapper):
                 metric_description = metric_description.get_description()
                 self.result_metadata[0]["text"] = metric_description
 
-            description_html.append(
-                self._markdown_description_to_html(metric_description)
-            )
+            description_html.append(metric_description)
 
         description_html.append(
             f"""

validmind/vm_models/test/test.py

@@ -52,6 +52,9 @@ class Test(TestUtils):
                 "test_id is missing. It must be passed when initializing the test"
             )
         self._ref_id = str(uuid4())
+        self.key = (
+            self.test_id
+        )  # for backwards compatibility - figures really should get keyed automatically
 
         # TODO: add validation for required inputs
         if self.default_params is None:

validmind/vm_models/test/threshold_test.py

@@ -11,7 +11,7 @@ avoid confusion with the "tests" in the general data science/modeling sense.
 from dataclasses import dataclass
 from typing import ClassVar, List, Optional
 
-from ...utils import get_description_metadata
+from ...ai.test_descriptions import get_description_metadata
 from ..figure import Figure
 from .result_summary import ResultSummary, ResultTable
 from .result_wrapper import ThresholdTestResultWrapper

validmind/vm_models/test_suite/runner.py

@@ -83,11 +83,14 @@ class TestSuiteRunner:
                     test_configs = test_configs.get("params", {})
                 else:
                     if (test_configs) and ("params" not in test_configs):
-                        """[DEPRECATED] Deprecated method for setting test parameters directly in the 'config' parameter"""
-                        logger.info(
-                            "Setting test parameters directly in the 'config' parameter of the run_documentation_tests() method is deprecated. "
-                            'Instead, use the new format of the config: config = {"test_id": {"params": {...}, "inputs": {...}}}'
+                        # [DEPRECATED] This is the old way of setting test parameters
+                        msg = (
+                            "Setting test parameters directly in the 'config' parameter"
+                            " of the run_documentation_tests() method is deprecated. "
+                            "Instead, use the new format of the config: "
+                            'config = {"test_id": {"params": {...}, "inputs": {...}}}'
                         )
+                        logger.warning(msg)
 
                 test.load(inputs=inputs, context=self.context, config=test_configs)
 
@@ -145,14 +148,17 @@ class TestSuiteRunner:
 
             await asyncio.sleep(0.5)
 
-    def summarize(self):
+    def summarize(self, show_link: bool = True):
         if not is_notebook():
             return logger.info("Test suite done...")
 
+        self.pbar_description.value = "Collecting test results..."
+
         summary = TestSuiteSummary(
             title=self.suite.title,
             description=self.suite.description,
             sections=self.suite.sections,
+            show_link=show_link,
         )
         summary.display()
 
@@ -181,6 +187,6 @@ class TestSuiteRunner:
             run_async(self.log_results)
             run_async_check(self._check_progress)
 
-        self.summarize()
+        self.summarize(show_link=send)
 
         self._stop_progress_bar()

validmind/vm_models/test_suite/summary.py

@@ -35,8 +35,14 @@ class TestSuiteSectionSummary:
         self._build_summary()
 
     def _add_description(self):
-        description = f'<div class="result">{md_to_html(self.description)}</div>'
-        self._widgets.append(widgets.HTML(value=description))
+        if not self.description:
+            return
+
+        self._widgets.append(
+            widgets.HTML(
+                value=f'<div class="result">{md_to_html(self.description)}</div>'
+            )
+        )
 
     def _add_tests_summary(self):
         children = []
@@ -45,9 +51,9 @@ class TestSuiteSectionSummary:
         for test in self.tests:
             children.append(test.result.to_widget())
             titles.append(
-                f"❌ {test.result.name}: {test.title} ({test.test_id})"
+                f"❌ {test.result.name}: {test.name} ({test.test_id})"
                 if isinstance(test.result, FailedResultWrapper)
-                else f"{test.result.name}: {test.title} ({test.test_id})"
+                else f"{test.result.name}: {test.name} ({test.test_id})"
             )
 
         self._widgets.append(widgets.Accordion(children=children, titles=titles))
@@ -71,6 +77,7 @@ class TestSuiteSummary:
     title: str
     description: str
     sections: List[TestSuiteSection]
+    show_link: bool = True
 
     _widgets: List[widgets.Widget] = None
 
@@ -100,8 +107,11 @@ class TestSuiteSummary:
         self._widgets.append(widgets.HTML(value=results_link))
 
     def _add_description(self):
-        description = f'<div class="result">{md_to_html(self.description)}</div>'
-        self._widgets.append(widgets.HTML(value=description))
+        self._widgets.append(
+            widgets.HTML(
+                value=f'<div class="result">{md_to_html(self.description)}</div>'
+            )
+        )
 
     def _add_sections_summary(self):
         children = []
@@ -145,7 +155,8 @@ class TestSuiteSummary:
         self._widgets = []
 
         self._add_title()
-        self._add_results_link()
+        if self.show_link:
+            self._add_results_link()
         self._add_description()
         if len(self.sections) == 1:
             self._add_top_level_section_summary()

validmind/vm_models/test_suite/test.py

@@ -21,6 +21,7 @@ class TestSuiteTest:
 
     test_id: str
     output_template: str = None
+    name: str = None
 
     _test_class: Test = None
     _test_instance: Test = None
@@ -39,6 +40,8 @@ class TestSuiteTest:
             self.test_id = test_id_or_obj["id"]
             self.output_template = test_id_or_obj.get("output_template")
 
+        self.name = test_id_to_name(self.test_id)
+
         try:
             self._test_class = load_test_class(self.test_id)
         except LoadTestError as e:
@@ -52,14 +55,6 @@ class TestSuiteTest:
             # since _test_class is None
             logger.error(f"Failed to load test '{self.test_id}': {e}")
 
-    @property
-    def title(self):
-        return test_id_to_name(self.test_id)
-
-    @property
-    def name(self):
-        return self._test_class.name
-
     @property
     def test_type(self):
         return self._test_class.test_type
@@ -86,12 +81,12 @@ class TestSuiteTest:
             )
         except Exception as e:
             logger.error(
-                f"Failed to load test '{self._test_class.name}': "
+                f"Failed to load test '{self.test_id}': "
                 f"({e.__class__.__name__}) {e}"
             )
             self.result = FailedResultWrapper(
                 error=e,
-                message=f"Failed to load test '{self.test_id}'",
+                message=f"Failed to load test '{self.name}'",
                 result_id=self.test_id,
             )
 
@@ -107,7 +102,7 @@ class TestSuiteTest:
             # run the test and log the performance if LOG_LEVEL is set to DEBUG
             log_performance(
                 func=self._test_instance.run,
-                name=self._test_instance.name,
+                name=self.test_id,
                 logger=logger,
             )()  # this is a decorator so we need to call it
 
@@ -116,14 +111,13 @@ class TestSuiteTest:
                 raise e  # Re-raise the exception if we are in fail fast mode
 
             logger.error(
-                f"Failed to run test '{self._test_instance.name}': "
-                f"({e.__class__.__name__}) {e}"
+                f"Failed to run test '{self.test_id}': " f"({e.__class__.__name__}) {e}"
             )
             self.result = FailedResultWrapper(
                 name=f"Failed {self._test_instance.test_type}",
                 error=e,
-                message=f"Failed to run '{self._test_instance.name}'",
-                result_id=self._test_instance.name,
+                message=f"Failed to run '{self.name}'",
+                result_id=self.test_id,
             )
 
             return
@@ -132,8 +126,8 @@ class TestSuiteTest:
             self.result = FailedResultWrapper(
                 name=f"Failed {self._test_instance.test_type}",
                 error=None,
-                message=f"'{self._test_instance.name}' did not return a result",
-                result_id=self._test_instance.name,
+                message=f"'{self.name}' did not return a result",
+                result_id=self.test_id,
             )
 
             return
@@ -142,9 +136,8 @@ class TestSuiteTest:
             self.result = FailedResultWrapper(
                 name=f"Failed {self._test_instance.test_type}",
                 error=None,
-                message=f"'{self._test_instance.name}' returned an invalid result: "
-                f"{self._test_instance.result}",
-                result_id=self._test_instance.name,
+                message=f"{self.name} returned an invalid result: {self._test_instance.result}",
+                result_id=self.test_id,
             )
 
             return

{validmind-2.2.5.dist-info → validmind-2.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.2.5
+Version: 2.3.1
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez