validmind 2.8.12__py3-none-any.whl → 2.8.22__py3-none-any.whl

validmind/vm_models/dataset/utils.py

@@ -45,11 +45,11 @@ class ExtraColumns:
         )
 
     def __contains__(self, key):
-        """Allow checking if a key is `in` the extra columns"""
+        """Allow checking if a key is `in` the extra columns."""
         return key in self.flatten()
 
     def flatten(self) -> List[str]:
-        """Get a list of all column names"""
+        """Get a list of all column names."""
         return [
             self.group_by_column,
             *self.extras,
@@ -78,13 +78,14 @@ class ExtraColumns:
 
 
 def as_df(series_or_frame: Union[pd.Series, pd.DataFrame]) -> pd.DataFrame:
+    """Convert a pandas Series or DataFrame to a DataFrame."""
     if isinstance(series_or_frame, pd.Series):
         return series_or_frame.to_frame()
     return series_or_frame
 
 
 def _is_probabilties(output):
-    """Check if the output from the predict method is probabilities."""
+    """Check if the output is a probability array."""
     if not isinstance(output, np.ndarray) or output.ndim > 1:
         return False
 
@@ -98,6 +99,7 @@ def _is_probabilties(output):
 
 
 def compute_predictions(model, X, **kwargs) -> tuple:
+    """Compute predictions and probabilities for a model."""
     probability_values = None
 
     try:

validmind/vm_models/figure.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-Figure objects track the figure schema supported by the ValidMind API
+Figure objects track the figure schema supported by the ValidMind API.
 """
 
 import base64
@@ -38,7 +38,7 @@ def create_figure(
     key: str,
     ref_id: str,
 ) -> "Figure":
-    """Create a VM Figure object from a raw figure object"""
+    """Create a VM Figure object from a raw figure object."""
     if is_matplotlib_figure(figure) or is_plotly_figure(figure) or is_png_image(figure):
         return Figure(key=key, figure=figure, ref_id=ref_id)
 
@@ -48,7 +48,7 @@ def create_figure(
 @dataclass
 class Figure:
     """
-    Figure objects track the schema supported by the ValidMind API
+    Figure objects track the schema supported by the ValidMind API.
     """
 
     key: str
@@ -115,7 +115,7 @@ class Figure:
 
     def serialize(self):
         """
-        Serializes the Figure to a dictionary so it can be sent to the API
+        Serializes the Figure to a dictionary so it can be sent to the API.
         """
         return {
             "type": self._type,
@@ -125,7 +125,7 @@ class Figure:
 
     def _get_b64_url(self):
         """
-        Returns a base64 encoded URL for the figure
+        Returns a base64 encoded URL for the figure.
         """
         if is_matplotlib_figure(self.figure):
             buffer = BytesIO()
@@ -152,7 +152,7 @@ class Figure:
         )
 
     def serialize_files(self):
-        """Creates a `requests`-compatible files object to be sent to the API"""
+        """Creates a `requests`-compatible files object to be sent to the API."""
         if is_matplotlib_figure(self.figure):
             buffer = BytesIO()
             self.figure.savefig(buffer, bbox_inches="tight")

validmind/vm_models/input.py

@@ -5,27 +5,28 @@
 """Base class for ValidMind Input types"""
 
 from abc import ABC
+from typing import Any, Dict
 
 
 class VMInput(ABC):
     """
-    Base class for ValidMind Input types
+    Base class for ValidMind Input types.
     """
 
-    def with_options(self, **kwargs) -> "VMInput":
+    def with_options(self, **kwargs: Dict[str, Any]) -> "VMInput":
         """
         Allows for setting options on the input object that are passed by the user
-        when using the input to run a test or set of tests
+        when using the input to run a test or set of tests.
 
         To allow options, just override this method in the subclass (see VMDataset)
         and ensure that it returns a new instance of the input with the specified options
         set.
 
         Args:
-            **kwargs: Arbitrary keyword arguments that will be passed to the input object
+            **kwargs: Arbitrary keyword arguments that will be passed to the input object.
 
         Returns:
-            VMInput: A new instance of the input with the specified options set
+            VMInput: A new instance of the input with the specified options set.
         """
         if kwargs:
             raise NotImplementedError("This type of input does not support options")

validmind/vm_models/model.py

@@ -40,7 +40,7 @@ R_MODEL_METHODS = [
 
 
 class ModelTask(Enum):
-    """Model task enums"""
+    """Model task enums."""
 
     # TODO: add more tasks
     CLASSIFICATION = "classification"
@@ -67,7 +67,7 @@ class ModelPipeline:
 @dataclass
 class ModelAttributes:
     """
-    Model attributes definition
+    Model attributes definition.
     """
 
     architecture: str = None
@@ -79,7 +79,7 @@ class ModelAttributes:
     @classmethod
     def from_dict(cls, data):
         """
-        Creates a ModelAttributes instance from a dictionary
+        Creates a ModelAttributes instance from a dictionary.
         """
         return cls(
             architecture=data.get("architecture"),
@@ -235,8 +235,8 @@ def is_model_metadata(model):
     Checks if the model is a dictionary containing metadata about a model.
     We want to check if the metadata dictionary contains at least the following keys:
 
-    - architecture
-    - language
+    - Architecture
+    - Language
     """
     if not isinstance(model, dict):
         return False

validmind/vm_models/result/result.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-Result Objects for test results
+Result objects for test results
 """
 import asyncio
 import json
@@ -19,6 +19,7 @@ from ipywidgets import HTML, VBox
 
 from ... import api_client
 from ...ai.utils import DescriptionFuture
+from ...errors import InvalidParameterError
 from ...logging import get_logger
 from ...utils import (
     HumanReadableEncoder,
@@ -43,15 +44,15 @@ logger = get_logger(__name__)
 
 
 class RawData:
-    """Holds raw data for a test result"""
+    """Holds raw data for a test result."""
 
-    def __init__(self, log: bool = False, **kwargs):
-        """Create a new RawData object
+    def __init__(self, log: bool = False, **kwargs: Any) -> None:
+        """Create a new RawData object.
 
         Args:
-            log (bool): If True, log the raw data to ValidMind
-            **kwargs: Keyword arguments to set as attributes e.g.
-                `RawData(log=True, dataset_duplicates=df_duplicates)`
+            log (bool): If True, log the raw data to ValidMind.
+            **kwargs: Keyword arguments to set as attributes, such as
+                `RawData(log=True, dataset_duplicates=df_duplicates)`.
         """
         self.log = log
 
@@ -61,8 +62,16 @@ class RawData:
     def __repr__(self) -> str:
         return f"RawData({', '.join(self.__dict__.keys())})"
 
-    def inspect(self, show: bool = True):
-        """Inspect the raw data"""
+    def inspect(self, show: bool = True) -> Optional[Dict[str, Any]]:
+        """Inspect the raw data.
+
+        Args:
+            show (bool): If True, print the raw data. If False, return it.
+
+        Returns:
+            Optional[Dict[str, Any]]: If True, print the raw data and return None. If
+                False, return the raw data dictionary.
+        """
         raw_data = {
             key: getattr(self, key)
             for key in self.__dict__
@@ -73,15 +82,21 @@ class RawData:
             return raw_data
 
         print(json.dumps(raw_data, indent=2, cls=HumanReadableEncoder))
+        return None
 
-    def serialize(self):
+    def serialize(self) -> Dict[str, Any]:
+        """Serialize the raw data to a dictionary
+
+        Returns:
+            Dict[str, Any]: The serialized raw data
+        """
         return {key: getattr(self, key) for key in self.__dict__}
 
 
 @dataclass
 class ResultTable:
     """
-    A dataclass that holds the table summary of result
+    A dataclass that holds the table summary of result.
     """
 
     data: Union[List[Any], pd.DataFrame]
@@ -110,33 +125,33 @@ class ResultTable:
 
 @dataclass
 class Result:
-    """Base Class for test suite results"""
+    """Base Class for test suite results."""
 
     result_id: str = None
     name: str = None
 
     def __str__(self) -> str:
-        """May be overridden by subclasses"""
+        """May be overridden by subclasses."""
         return self.__class__.__name__
 
     @abstractmethod
     def to_widget(self):
-        """Create an ipywdiget representation of the result... Must be overridden by subclasses"""
+        """Create an ipywidget representation of the result... Must be overridden by subclasses."""
         raise NotImplementedError
 
     @abstractmethod
     def log(self):
-        """Log the result... Must be overridden by subclasses"""
+        """Log the result... Must be overridden by subclasses."""
         raise NotImplementedError
 
     def show(self):
-        """Display the result... May be overridden by subclasses"""
+        """Display the result... May be overridden by subclasses."""
         display(self.to_widget())
 
 
 @dataclass
 class ErrorResult(Result):
-    """Result for test suites that fail to load or run properly"""
+    """Result for test suites that fail to load or run properly."""
 
     name: str = "Failed Test"
     error: Exception = None
@@ -154,7 +169,7 @@ class ErrorResult(Result):
 
 @dataclass
 class TestResult(Result):
-    """Test result"""
+    """Test result."""
 
     name: str = "Test Result"
     ref_id: str = None
@@ -232,12 +247,12 @@ class TestResult(Result):
         table: Union[ResultTable, pd.DataFrame, List[Dict[str, Any]]],
         title: Optional[str] = None,
     ):
-        """Add a new table to the result
+        """Add a new table to the result.
 
         Args:
-            table (Union[ResultTable, pd.DataFrame, List[Dict[str, Any]]]): The table to add
+            table (Union[ResultTable, pd.DataFrame, List[Dict[str, Any]]]): The table to add.
             title (Optional[str]): The title of the table (can optionally be provided for
-                pd.DataFrame and List[Dict[str, Any]] tables)
+                pd.DataFrame and List[Dict[str, Any]] tables).
         """
         if self.tables is None:
             self.tables = []
@@ -248,10 +263,10 @@ class TestResult(Result):
         self.tables.append(table)
 
     def remove_table(self, index: int):
-        """Remove a table from the result by index
+        """Remove a table from the result by index.
 
         Args:
-            index (int): The index of the table to remove (default is 0)
+            index (int): The index of the table to remove (default is 0).
         """
         if self.tables is None:
             return
@@ -267,14 +282,19 @@ class TestResult(Result):
             bytes,
             Figure,
         ],
-    ):
-        """Add a new figure to the result
+    ) -> None:
+        """Add a new figure to the result.
 
         Args:
-            figure (Union[matplotlib.figure.Figure, go.Figure, go.FigureWidget,
-                bytes, Figure]): The figure to add (can be either a VM Figure object,
-                a raw figure object from the supported libraries, or a png image as
-                raw bytes)
+            figure: The figure to add. Can be one of:
+                - matplotlib.figure.Figure: A matplotlib figure
+                - plotly.graph_objs.Figure: A plotly figure
+                - plotly.graph_objs.FigureWidget: A plotly figure widget
+                - bytes: A PNG image as raw bytes
+                - validmind.vm_models.figure.Figure: A ValidMind figure object.
+
+        Returns:
+            None.
         """
         if self.figures is None:
             self.figures = []
@@ -293,10 +313,10 @@ class TestResult(Result):
         self.figures.append(figure)
 
     def remove_figure(self, index: int = 0):
-        """Remove a figure from the result by index
+        """Remove a figure from the result by index.
 
         Args:
-            index (int): The index of the figure to remove (default is 0)
+            index (int): The index of the figure to remove (default is 0).
         """
         if self.figures is None:
             return
@@ -332,7 +352,7 @@ class TestResult(Result):
 
     @classmethod
     def _get_client_config(cls):
-        """Get the client config, loading it if not cached"""
+        """Get the client config, loading it if not cached."""
         if cls._client_config_cache is None:
             api_client.reload()
             cls._client_config_cache = api_client.client_config
@@ -350,7 +370,7 @@ class TestResult(Result):
         return cls._client_config_cache
 
     def check_result_id_exist(self):
-        """Check if the result_id exists in any test block across all sections"""
+        """Check if the result_id exists in any test block across all sections."""
         client_config = self._get_client_config()
 
         # Iterate through all sections
@@ -371,7 +391,7 @@ class TestResult(Result):
     def _validate_section_id_for_block(
         self, section_id: str, position: Union[int, None] = None
     ):
-        """Validate the section_id exits on the template before logging"""
+        """Validate the section_id exits on the template before logging."""
         client_config = self._get_client_config()
         found = False
 
@@ -410,7 +430,7 @@ class TestResult(Result):
                 )
 
     def serialize(self):
-        """Serialize the result for the API"""
+        """Serialize the result for the API."""
         return {
             "test_name": self.result_id,
             "title": self.title,
@@ -423,10 +443,16 @@ class TestResult(Result):
         }
 
     async def log_async(
-        self, section_id: str = None, position: int = None, unsafe: bool = False
+        self,
+        section_id: str = None,
+        position: int = None,
+        config: Dict[str, bool] = None,
     ):
         tasks = []  # collect tasks to run in parallel (async)
 
+        # Default empty dict if None
+        config = config or {}
+
         if self.metric is not None:
             # metrics are logged as separate entities
             tasks.append(
@@ -438,12 +464,13 @@ class TestResult(Result):
                 )
             )
 
-        if self.tables or self.figures:
+        if self.tables or self.figures or self.description:
             tasks.append(
                 api_client.alog_test_result(
                     result=self.serialize(),
                     section_id=section_id,
                     position=position,
+                    config=config,
                 )
             )
 
@@ -467,17 +494,32 @@ class TestResult(Result):
 
         return await asyncio.gather(*tasks)
 
-    def log(self, section_id: str = None, position: int = None, unsafe: bool = False):
-        """Log the result to ValidMind
+    def log(
+        self,
+        section_id: str = None,
+        position: int = None,
+        unsafe: bool = False,
+        config: Dict[str, bool] = None,
+    ):
+        """Log the result to ValidMind.
 
         Args:
             section_id (str): The section ID within the model document to insert the
-                test result
+                test result.
             position (int): The position (index) within the section to insert the test
-                result
+                result.
             unsafe (bool): If True, log the result even if it contains sensitive data
-                i.e. raw data from input datasets
+                i.e. raw data from input datasets.
+            config (Dict[str, bool]): Configuration options for displaying the test result.
+                Available config options:
+                - hideTitle: Hide the title in the document view
+                - hideText: Hide the description text in the document view
+                - hideParams: Hide the parameters in the document view
+                - hideTables: Hide tables in the document view
+                - hideFigures: Hide figures in the document view
         """
+        if config:
+            self.validate_log_config(config)
 
         self.check_result_id_exist()
 
@@ -488,4 +530,41 @@ class TestResult(Result):
         if section_id:
             self._validate_section_id_for_block(section_id, position)
 
-        run_async(self.log_async, section_id=section_id, position=position)
+        run_async(
+            self.log_async,
+            section_id=section_id,
+            position=position,
+            config=config,
+        )
+
+    def validate_log_config(self, config: Dict[str, bool]):
+        """Validate the configuration options for logging a test result
+
+        Args:
+            config (Dict[str, bool]): Configuration options to validate
+
+        Raises:
+            InvalidParameterError: If config contains invalid keys or non-boolean values
+        """
+        valid_keys = {
+            "hideTitle",
+            "hideText",
+            "hideParams",
+            "hideTables",
+            "hideFigures",
+        }
+        invalid_keys = set(config.keys()) - valid_keys
+        if invalid_keys:
+            raise InvalidParameterError(
+                f"Invalid config keys: {', '.join(invalid_keys)}. "
+                f"Valid keys are: {', '.join(valid_keys)}"
+            )
+
+        # Ensure all values are boolean
+        non_bool_keys = [
+            key for key, value in config.items() if not isinstance(value, bool)
+        ]
+        if non_bool_keys:
+            raise InvalidParameterError(
+                f"Values for config keys must be boolean. Non-boolean values found for keys: {', '.join(non_bool_keys)}"
+            )

validmind/vm_models/result/utils.py

@@ -28,7 +28,7 @@ _result_template = None
 
 
 def get_result_template():
-    """Get the jinja html template for rendering test results"""
+    """Get the Jinja2 HTML template for rendering test results."""
     global _result_template
 
     if _result_template is None:
@@ -39,7 +39,7 @@ def get_result_template():
 
 
 async def update_metadata(content_id: str, text: str, _json: Union[Dict, List] = None):
-    """Create or Update a Metadata Object"""
+    """Create or update a metadata object."""
     parts = content_id.split("::")
     content_id = parts[0]
     revision_name = parts[1] if len(parts) > 1 else None
@@ -53,7 +53,7 @@ async def update_metadata(content_id: str, text: str, _json: Union[Dict, List] =
 
 
 def check_for_sensitive_data(data: pd.DataFrame, inputs: List[VMInput]):
-    """Check if a table contains raw data from input datasets"""
+    """Check if the data contains sensitive information from input datasets."""
     dataset_columns = {
         col: len(input_obj.df)
         for input_obj in inputs
@@ -77,7 +77,7 @@ def check_for_sensitive_data(data: pd.DataFrame, inputs: List[VMInput]):
 
 
 def tables_to_widgets(tables: List["ResultTable"]):
-    """Convert summary (list of json tables) into a list of ipywidgets"""
+    """Convert a list of tables to ipywidgets."""
     widgets = [
         HTML("<h3>Tables</h3>"),
     ]
@@ -128,7 +128,7 @@ def tables_to_widgets(tables: List["ResultTable"]):
 
 
 def figures_to_widgets(figures: List[Figure]) -> list:
-    """Plot figures to a ipywidgets GridBox"""
+    """Convert a list of figures to ipywidgets."""
     num_columns = 2 if len(figures) > 1 else 1
 
     plot_widgets = GridBox(

validmind/vm_models/test_suite/__init__.py

@@ -0,0 +1,5 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Test suite module."""

validmind/vm_models/test_suite/runner.py

@@ -17,7 +17,7 @@ logger = get_logger(__name__)
 
 class TestSuiteRunner:
     """
-    Runs a test suite
+    Runs a test suite.
     """
 
     suite: TestSuite = None
@@ -36,7 +36,7 @@ class TestSuiteRunner:
         self._load_config(inputs)
 
     def _load_config(self, inputs: dict = None):
-        """Splits the config into a global config and test configs"""
+        """Splits the config into a global config and test configs."""
         self._test_configs = {
             test.test_id: {"inputs": inputs or {}} for test in self.suite.get_tests()
         }
@@ -59,7 +59,7 @@ class TestSuiteRunner:
 
     def _start_progress_bar(self, send: bool = True):
         """
-        Initializes the progress bar elements
+        Initializes the progress bar elements.
         """
         # TODO: make this work for when user runs only a section of the test suite
         # if we are sending then there is a task for each test and logging its result
@@ -76,7 +76,7 @@ class TestSuiteRunner:
         self.pbar.close()
 
     async def log_results(self):
-        """Logs the results of the test suite to ValidMind
+        """Logs the results of the test suite to ValidMind.
 
         This method will be called after the test suite has been run and all results have been
         collected. This method will log the results to ValidMind.
@@ -127,7 +127,7 @@ class TestSuiteRunner:
         summary.display()
 
     def run(self, send: bool = True, fail_fast: bool = False):
-        """Runs the test suite, renders the summary and sends the results to ValidMind
+        """Runs the test suite, renders the summary and sends the results to ValidMind.
 
         Args:
             send (bool, optional): Whether to send the results to ValidMind.