validmind 2.8.12__py3-none-any.whl → 2.8.20__py3-none-any.whl

validmind/vm_models/result/result.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-Result Objects for test results
+Result objects for test results
 """
 import asyncio
 import json
@@ -19,6 +19,7 @@ from ipywidgets import HTML, VBox
 
 from ... import api_client
 from ...ai.utils import DescriptionFuture
+from ...errors import InvalidParameterError
 from ...logging import get_logger
 from ...utils import (
     HumanReadableEncoder,
@@ -43,15 +44,15 @@ logger = get_logger(__name__)
 
 
 class RawData:
-    """Holds raw data for a test result"""
+    """Holds raw data for a test result."""
 
-    def __init__(self, log: bool = False, **kwargs):
-        """Create a new RawData object
+    def __init__(self, log: bool = False, **kwargs: Any) -> None:
+        """Create a new RawData object.
 
         Args:
-            log (bool): If True, log the raw data to ValidMind
-            **kwargs: Keyword arguments to set as attributes e.g.
-                `RawData(log=True, dataset_duplicates=df_duplicates)`
+            log (bool): If True, log the raw data to ValidMind.
+            **kwargs: Keyword arguments to set as attributes, such as
+                `RawData(log=True, dataset_duplicates=df_duplicates)`.
         """
         self.log = log
 
@@ -61,8 +62,16 @@ class RawData:
     def __repr__(self) -> str:
         return f"RawData({', '.join(self.__dict__.keys())})"
 
-    def inspect(self, show: bool = True):
-        """Inspect the raw data"""
+    def inspect(self, show: bool = True) -> Optional[Dict[str, Any]]:
+        """Inspect the raw data.
+
+        Args:
+            show (bool): If True, print the raw data. If False, return it.
+
+        Returns:
+            Optional[Dict[str, Any]]: If True, print the raw data and return None. If
+                False, return the raw data dictionary.
+        """
         raw_data = {
             key: getattr(self, key)
             for key in self.__dict__
@@ -73,15 +82,21 @@ class RawData:
             return raw_data
 
         print(json.dumps(raw_data, indent=2, cls=HumanReadableEncoder))
+        return None
 
-    def serialize(self):
+    def serialize(self) -> Dict[str, Any]:
+        """Serialize the raw data to a dictionary
+
+        Returns:
+            Dict[str, Any]: The serialized raw data
+        """
         return {key: getattr(self, key) for key in self.__dict__}
 
 
 @dataclass
 class ResultTable:
     """
-    A dataclass that holds the table summary of result
+    A dataclass that holds the table summary of result.
     """
 
     data: Union[List[Any], pd.DataFrame]
@@ -110,33 +125,33 @@ class ResultTable:
 
 @dataclass
 class Result:
-    """Base Class for test suite results"""
+    """Base Class for test suite results."""
 
     result_id: str = None
     name: str = None
 
     def __str__(self) -> str:
-        """May be overridden by subclasses"""
+        """May be overridden by subclasses."""
         return self.__class__.__name__
 
     @abstractmethod
     def to_widget(self):
-        """Create an ipywdiget representation of the result... Must be overridden by subclasses"""
+        """Create an ipywidget representation of the result... Must be overridden by subclasses."""
         raise NotImplementedError
 
     @abstractmethod
     def log(self):
-        """Log the result... Must be overridden by subclasses"""
+        """Log the result... Must be overridden by subclasses."""
         raise NotImplementedError
 
     def show(self):
-        """Display the result... May be overridden by subclasses"""
+        """Display the result... May be overridden by subclasses."""
         display(self.to_widget())
 
 
 @dataclass
 class ErrorResult(Result):
-    """Result for test suites that fail to load or run properly"""
+    """Result for test suites that fail to load or run properly."""
 
     name: str = "Failed Test"
     error: Exception = None
@@ -154,7 +169,7 @@ class ErrorResult(Result):
 
 @dataclass
 class TestResult(Result):
-    """Test result"""
+    """Test result."""
 
     name: str = "Test Result"
     ref_id: str = None
@@ -232,12 +247,12 @@ class TestResult(Result):
         table: Union[ResultTable, pd.DataFrame, List[Dict[str, Any]]],
         title: Optional[str] = None,
     ):
-        """Add a new table to the result
+        """Add a new table to the result.
 
         Args:
-            table (Union[ResultTable, pd.DataFrame, List[Dict[str, Any]]]): The table to add
+            table (Union[ResultTable, pd.DataFrame, List[Dict[str, Any]]]): The table to add.
             title (Optional[str]): The title of the table (can optionally be provided for
-                pd.DataFrame and List[Dict[str, Any]] tables)
+                pd.DataFrame and List[Dict[str, Any]] tables).
         """
         if self.tables is None:
             self.tables = []
@@ -248,10 +263,10 @@ class TestResult(Result):
         self.tables.append(table)
 
     def remove_table(self, index: int):
-        """Remove a table from the result by index
+        """Remove a table from the result by index.
 
         Args:
-            index (int): The index of the table to remove (default is 0)
+            index (int): The index of the table to remove (default is 0).
         """
         if self.tables is None:
             return
@@ -267,14 +282,19 @@ class TestResult(Result):
             bytes,
             Figure,
         ],
-    ):
-        """Add a new figure to the result
+    ) -> None:
+        """Add a new figure to the result.
 
         Args:
-            figure (Union[matplotlib.figure.Figure, go.Figure, go.FigureWidget,
-                bytes, Figure]): The figure to add (can be either a VM Figure object,
-                a raw figure object from the supported libraries, or a png image as
-                raw bytes)
+            figure: The figure to add. Can be one of:
+                - matplotlib.figure.Figure: A matplotlib figure
+                - plotly.graph_objs.Figure: A plotly figure
+                - plotly.graph_objs.FigureWidget: A plotly figure widget
+                - bytes: A PNG image as raw bytes
+                - validmind.vm_models.figure.Figure: A ValidMind figure object.
+
+        Returns:
+            None.
         """
         if self.figures is None:
             self.figures = []
@@ -293,10 +313,10 @@ class TestResult(Result):
         self.figures.append(figure)
 
     def remove_figure(self, index: int = 0):
-        """Remove a figure from the result by index
+        """Remove a figure from the result by index.
 
         Args:
-            index (int): The index of the figure to remove (default is 0)
+            index (int): The index of the figure to remove (default is 0).
         """
         if self.figures is None:
             return
@@ -332,7 +352,7 @@ class TestResult(Result):
 
     @classmethod
     def _get_client_config(cls):
-        """Get the client config, loading it if not cached"""
+        """Get the client config, loading it if not cached."""
         if cls._client_config_cache is None:
             api_client.reload()
             cls._client_config_cache = api_client.client_config
@@ -350,7 +370,7 @@ class TestResult(Result):
         return cls._client_config_cache
 
     def check_result_id_exist(self):
-        """Check if the result_id exists in any test block across all sections"""
+        """Check if the result_id exists in any test block across all sections."""
         client_config = self._get_client_config()
 
         # Iterate through all sections
@@ -371,7 +391,7 @@ class TestResult(Result):
     def _validate_section_id_for_block(
         self, section_id: str, position: Union[int, None] = None
     ):
-        """Validate the section_id exits on the template before logging"""
+        """Validate the section_id exits on the template before logging."""
         client_config = self._get_client_config()
         found = False
 
@@ -410,7 +430,7 @@ class TestResult(Result):
                 )
 
     def serialize(self):
-        """Serialize the result for the API"""
+        """Serialize the result for the API."""
         return {
             "test_name": self.result_id,
             "title": self.title,
@@ -423,10 +443,16 @@ class TestResult(Result):
         }
 
     async def log_async(
-        self, section_id: str = None, position: int = None, unsafe: bool = False
+        self,
+        section_id: str = None,
+        position: int = None,
+        config: Dict[str, bool] = None,
     ):
         tasks = []  # collect tasks to run in parallel (async)
 
+        # Default empty dict if None
+        config = config or {}
+
         if self.metric is not None:
             # metrics are logged as separate entities
             tasks.append(
@@ -438,12 +464,13 @@ class TestResult(Result):
                 )
             )
 
-        if self.tables or self.figures:
+        if self.tables or self.figures or self.description:
             tasks.append(
                 api_client.alog_test_result(
                     result=self.serialize(),
                     section_id=section_id,
                     position=position,
+                    config=config,
                 )
             )
 
@@ -467,17 +494,32 @@ class TestResult(Result):
 
         return await asyncio.gather(*tasks)
 
-    def log(self, section_id: str = None, position: int = None, unsafe: bool = False):
-        """Log the result to ValidMind
+    def log(
+        self,
+        section_id: str = None,
+        position: int = None,
+        unsafe: bool = False,
+        config: Dict[str, bool] = None,
+    ):
+        """Log the result to ValidMind.
 
         Args:
             section_id (str): The section ID within the model document to insert the
-                test result
+                test result.
             position (int): The position (index) within the section to insert the test
-                result
+                result.
             unsafe (bool): If True, log the result even if it contains sensitive data
-                i.e. raw data from input datasets
+                i.e. raw data from input datasets.
+            config (Dict[str, bool]): Configuration options for displaying the test result.
+                Available config options:
+                - hideTitle: Hide the title in the document view
+                - hideText: Hide the description text in the document view
+                - hideParams: Hide the parameters in the document view
+                - hideTables: Hide tables in the document view
+                - hideFigures: Hide figures in the document view
         """
+        if config:
+            self.validate_log_config(config)
 
         self.check_result_id_exist()
 
@@ -488,4 +530,41 @@ class TestResult(Result):
         if section_id:
             self._validate_section_id_for_block(section_id, position)
 
-        run_async(self.log_async, section_id=section_id, position=position)
+        run_async(
+            self.log_async,
+            section_id=section_id,
+            position=position,
+            config=config,
+        )
+
+    def validate_log_config(self, config: Dict[str, bool]):
+        """Validate the configuration options for logging a test result
+
+        Args:
+            config (Dict[str, bool]): Configuration options to validate
+
+        Raises:
+            InvalidParameterError: If config contains invalid keys or non-boolean values
+        """
+        valid_keys = {
+            "hideTitle",
+            "hideText",
+            "hideParams",
+            "hideTables",
+            "hideFigures",
+        }
+        invalid_keys = set(config.keys()) - valid_keys
+        if invalid_keys:
+            raise InvalidParameterError(
+                f"Invalid config keys: {', '.join(invalid_keys)}. "
+                f"Valid keys are: {', '.join(valid_keys)}"
+            )
+
+        # Ensure all values are boolean
+        non_bool_keys = [
+            key for key, value in config.items() if not isinstance(value, bool)
+        ]
+        if non_bool_keys:
+            raise InvalidParameterError(
+                f"Values for config keys must be boolean. Non-boolean values found for keys: {', '.join(non_bool_keys)}"
+            )

validmind/vm_models/result/utils.py

@@ -28,7 +28,7 @@ _result_template = None
 
 
 def get_result_template():
-    """Get the jinja html template for rendering test results"""
+    """Get the Jinja2 HTML template for rendering test results."""
     global _result_template
 
     if _result_template is None:
@@ -39,7 +39,7 @@ def get_result_template():
 
 
 async def update_metadata(content_id: str, text: str, _json: Union[Dict, List] = None):
-    """Create or Update a Metadata Object"""
+    """Create or update a metadata object."""
     parts = content_id.split("::")
     content_id = parts[0]
     revision_name = parts[1] if len(parts) > 1 else None
@@ -53,7 +53,7 @@ async def update_metadata(content_id: str, text: str, _json: Union[Dict, List] =
 
 
 def check_for_sensitive_data(data: pd.DataFrame, inputs: List[VMInput]):
-    """Check if a table contains raw data from input datasets"""
+    """Check if the data contains sensitive information from input datasets."""
     dataset_columns = {
         col: len(input_obj.df)
         for input_obj in inputs
@@ -77,7 +77,7 @@ def check_for_sensitive_data(data: pd.DataFrame, inputs: List[VMInput]):
 
 
 def tables_to_widgets(tables: List["ResultTable"]):
-    """Convert summary (list of json tables) into a list of ipywidgets"""
+    """Convert a list of tables to ipywidgets."""
     widgets = [
         HTML("<h3>Tables</h3>"),
     ]
@@ -128,7 +128,7 @@ def tables_to_widgets(tables: List["ResultTable"]):
 
 
 def figures_to_widgets(figures: List[Figure]) -> list:
-    """Plot figures to a ipywidgets GridBox"""
+    """Convert a list of figures to ipywidgets."""
     num_columns = 2 if len(figures) > 1 else 1
 
     plot_widgets = GridBox(

validmind/vm_models/test_suite/__init__.py

@@ -0,0 +1,5 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Test suite module."""

validmind/vm_models/test_suite/runner.py

@@ -17,7 +17,7 @@ logger = get_logger(__name__)
 
 class TestSuiteRunner:
     """
-    Runs a test suite
+    Runs a test suite.
     """
 
     suite: TestSuite = None
@@ -36,7 +36,7 @@ class TestSuiteRunner:
         self._load_config(inputs)
 
     def _load_config(self, inputs: dict = None):
-        """Splits the config into a global config and test configs"""
+        """Splits the config into a global config and test configs."""
         self._test_configs = {
             test.test_id: {"inputs": inputs or {}} for test in self.suite.get_tests()
         }
@@ -59,7 +59,7 @@ class TestSuiteRunner:
 
     def _start_progress_bar(self, send: bool = True):
         """
-        Initializes the progress bar elements
+        Initializes the progress bar elements.
         """
         # TODO: make this work for when user runs only a section of the test suite
         # if we are sending then there is a task for each test and logging its result
@@ -76,7 +76,7 @@ class TestSuiteRunner:
         self.pbar.close()
 
     async def log_results(self):
-        """Logs the results of the test suite to ValidMind
+        """Logs the results of the test suite to ValidMind.
 
         This method will be called after the test suite has been run and all results have been
         collected. This method will log the results to ValidMind.
@@ -127,7 +127,7 @@ class TestSuiteRunner:
         summary.display()
 
     def run(self, send: bool = True, fail_fast: bool = False):
-        """Runs the test suite, renders the summary and sends the results to ValidMind
+        """Runs the test suite, renders the summary and sends the results to ValidMind.
 
         Args:
             send (bool, optional): Whether to send the results to ValidMind.

validmind/vm_models/test_suite/summary.py

@@ -16,6 +16,7 @@ logger = get_logger(__name__)
 
 
 def id_to_name(id: str) -> str:
+    """Convert an ID to a human-readable name."""
     # replace underscores, hyphens etc with spaces
     name = id.replace("_", " ").replace("-", " ").replace(".", " ")
     # capitalize each word
@@ -26,6 +27,8 @@ def id_to_name(id: str) -> str:
 
 @dataclass
 class TestSuiteSectionSummary:
+    """Represents a summary of a test suite section."""
+
     tests: List[TestSuiteTest]
     description: Optional[str] = None
 
@@ -35,6 +38,7 @@ class TestSuiteSectionSummary:
         self._build_summary()
 
     def _add_description(self):
+        """Add the section description to the summary."""
         if not self.description:
             return
 
@@ -45,6 +49,7 @@ class TestSuiteSectionSummary:
         )
 
     def _add_tests_summary(self):
+        """Add the test results summary."""
         children = []
         titles = []
 
@@ -59,6 +64,7 @@ class TestSuiteSectionSummary:
         self._widgets.append(widgets.Accordion(children=children, titles=titles))
 
     def _build_summary(self):
+        """Build the complete summary."""
         self._widgets = []
 
         if self.description:
@@ -69,11 +75,14 @@ class TestSuiteSectionSummary:
         self.summary = widgets.VBox(self._widgets)
 
     def display(self):
+        """Display the summary."""
         display(self.summary)
 
 
 @dataclass
 class TestSuiteSummary:
+    """Represents a summary of a complete test suite."""
+
     title: str
     description: str
     sections: List[TestSuiteSection]
@@ -82,9 +91,11 @@ class TestSuiteSummary:
     _widgets: List[widgets.Widget] = None
 
     def __post_init__(self):
+        """Initialize the summary after the dataclass is created."""
         self._build_summary()
 
     def _add_title(self):
+        """Add the title to the summary."""
         title = f"""
         <h2>Test Suite Results: <i style="color: #DE257E">{self.title}</i></h2><hr>
         """.strip()
@@ -92,6 +103,7 @@ class TestSuiteSummary:
         self._widgets.append(widgets.HTML(value=title))
 
     def _add_results_link(self):
+        """Add a link to documentation on ValidMind."""
         # avoid circular import
         from ...api_client import get_api_host, get_api_model
 
@@ -99,14 +111,15 @@ class TestSuiteSummary:
         link = f"{ui_host}model-inventory/{get_api_model()}"
         results_link = f"""
         <h3>
-            Check out the updated documentation in your
-            <a href="{link}" target="_blank">ValidMind project</a>.
+            Check out the updated documentation on
+            <a href="{link}" target="_blank">ValidMind</a>.
         </h3>
         """.strip()
 
         self._widgets.append(widgets.HTML(value=results_link))
 
     def _add_description(self):
+        """Add the test suite description to the summary."""
         self._widgets.append(
             widgets.HTML(
                 value=f'<div class="result">{md_to_html(self.description)}</div>'
@@ -114,6 +127,7 @@ class TestSuiteSummary:
         )
 
     def _add_sections_summary(self):
+        """Append the section summary."""
         children = []
         titles = []
 
@@ -132,11 +146,13 @@ class TestSuiteSummary:
         self._widgets.append(widgets.Accordion(children=children, titles=titles))
 
     def _add_top_level_section_summary(self):
+        """Add the top-level section summary."""
         self._widgets.append(
             TestSuiteSectionSummary(tests=self.sections[0].tests).summary
         )
 
     def _add_footer(self):
+        """Add the footer."""
         footer = """
         <style>
             .result {
@@ -152,6 +168,7 @@ class TestSuiteSummary:
         self._widgets.append(widgets.HTML(value=footer))
 
     def _build_summary(self):
+        """Build the complete summary."""
         self._widgets = []
 
         self._add_title()
@@ -166,4 +183,5 @@ class TestSuiteSummary:
         self.summary = widgets.VBox(self._widgets)
 
     def display(self):
+        """Display the summary."""
         display(self.summary)

validmind/vm_models/test_suite/test.py

@@ -16,7 +16,7 @@ logger = get_logger(__name__)
 
 class TestSuiteTest:
     """
-    Wraps a 'Test' in a Test Suite and handles logic and state for that test
+    Wraps a 'Test' in a Test Suite and handles logic and state for that test.
     """
 
     test_id: str
@@ -28,10 +28,10 @@ class TestSuiteTest:
     _load_failed: bool = False
 
     def __init__(self, test_id_or_obj):
-        """Load the test class from the test id
+        """Load the test class from the test ID.
 
         Args:
-            test_id_or_obj (str): The test id or a dict with test id and other options
+            test_id_or_obj (str): The test ID or a dict with test ID and other options.
         """
         if isinstance(test_id_or_obj, str):
             self.test_id = test_id_or_obj
@@ -42,7 +42,7 @@ class TestSuiteTest:
         self.name = test_id_to_name(self.test_id)
 
     def get_default_config(self):
-        """Returns the default configuration for the test"""
+        """Returns the default configuration for the test."""
         try:
             test_func = load_test(self.test_id)
         except LoadTestError as e:
@@ -66,7 +66,7 @@ class TestSuiteTest:
         return config
 
     def run(self, fail_fast: bool = False, config: dict = None):
-        """Run the test"""
+        """Run the test."""
         if self._load_failed:
             return
 
@@ -110,7 +110,7 @@ class TestSuiteTest:
             )
 
     async def log_async(self):
-        """Log the result for this test to ValidMind"""
+        """Log the result for this test to ValidMind."""
         if not self.result:
             raise ValueError("Cannot log test result before running the test")