validmind 2.8.12__py3-none-any.whl → 2.8.22__py3-none-any.whl

validmind/errors.py

@@ -15,6 +15,8 @@ from typing import Optional
 
 
 class BaseError(Exception):
+    """Common base class for all non-exit exceptions."""
+
     def __init__(self, message=""):
         self.message = message
         super().__init__(self.message)
@@ -52,7 +54,7 @@ class MissingCacheResultsArgumentsError(BaseError):
 
 class MissingOrInvalidModelPredictFnError(BaseError):
     """
-    When the pytorch model is missing a predict function or its predict
+    When the PyTorch model is missing a predict function or its predict
     method does not have the expected arguments.
     """
 
@@ -71,7 +73,7 @@ class InvalidAPICredentialsError(APIRequestError):
     def description(self, *args, **kwargs):
         return (
             self.message
-            or "Invalid API credentials. Please ensure that you have provided the correct values for api_key and api_secret."
+            or "Invalid API credentials. Please ensure that you have provided the correct values for API_KEY and API_SECRET."
         )
 
 
@@ -115,7 +117,7 @@ class InvalidTestResultsError(APIRequestError):
 
 class InvalidTestParametersError(BaseError):
     """
-    When an invalid parameters for the test.
+    When invalid parameters are provided for the test.
     """
 
     pass
@@ -123,7 +125,15 @@ class InvalidTestParametersError(BaseError):
 
 class InvalidInputError(BaseError):
     """
-    When an invalid input object.
+    When an invalid input object is provided.
+    """
+
+    pass
+
+
+class InvalidParameterError(BaseError):
+    """
+    When an invalid parameter is provided.
     """
 
     pass
@@ -131,7 +141,7 @@ class InvalidInputError(BaseError):
 
 class InvalidTextObjectError(APIRequestError):
     """
-    When an invalid Metadat (Text) object is sent to the API.
+    When an invalid Metadata (Text) object is sent to the API.
     """
 
     pass
@@ -155,7 +165,7 @@ class InvalidXGBoostTrainedModelError(BaseError):
 
 class LoadTestError(BaseError):
     """
-    Exception raised when an error occurs while loading a test
+    Exception raised when an error occurs while loading a test.
     """
 
     def __init__(self, message: str, original_error: Optional[Exception] = None):
@@ -323,7 +333,7 @@ class SkipTestError(BaseError):
 def raise_api_error(error_string):
     """
     Safely try to parse JSON from the response message in case the API
-    returns a non-JSON string or if the API returns a non-standard error
+    returns a non-JSON string or if the API returns a non-standard error.
     """
     try:
         json_response = json.loads(error_string)

validmind/input_registry.py

@@ -29,7 +29,7 @@ class InputRegistry:
         if not input_obj:
             raise InvalidInputError(
                 f"There's no such input with given ID '{key}'. "
-                "Please pass valid input ID"
+                "Please pass valid input ID."
             )
         return input_obj
 

validmind/logging.py

@@ -2,11 +2,12 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-"""ValidMind logging module."""
+"""ValidMind logging module"""
 
 import logging
 import os
 import time
+from typing import Any, Awaitable, Callable, Dict, Optional, TypeVar
 
 import sentry_sdk
 from sentry_sdk.utils import event_from_exception, exc_info_from_error
@@ -16,8 +17,8 @@ from .__version__ import __version__
 __dsn = "https://48f446843657444aa1e2c0d716ef864b@o1241367.ingest.sentry.io/4505239625465856"
 
 
-def _get_log_level():
-    """Get the log level from the environment variable"""
+def _get_log_level() -> int:
+    """Get the log level from the environment variable."""
     log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
 
     if log_level_str not in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]:
@@ -26,8 +27,10 @@ def _get_log_level():
     return logging.getLevelName(log_level_str)
 
 
-def get_logger(name="validmind", log_level=None):
-    """Get a logger for the given module name"""
+def get_logger(
+    name: str = "validmind", log_level: Optional[int] = None
+) -> logging.Logger:
+    """Get a logger for the given module name."""
     formatter = logging.Formatter(
         fmt="%(asctime)s - %(levelname)s(%(name)s): %(message)s"
     )
@@ -52,18 +55,21 @@ def get_logger(name="validmind", log_level=None):
     return logger
 
 
-def init_sentry(server_config):
-    """Initialize Sentry SDK for sending logs back to ValidMind
+def init_sentry(server_config: Dict[str, Any]) -> None:
+    """Initialize Sentry SDK for sending logs back to ValidMind.
 
-    This will usually only be called by the api_client module to initialize the
-    sentry connection after the user calls `validmind.init()`. This is because the DSN
+    This will usually only be called by the API client module to initialize the
+    Sentry connection after the user calls `validmind.init()`. This is because the DSN
     and other config options will be returned by the API.
 
     Args:
-        config (dict): The config dictionary returned by the API
-            - send_logs (bool): Whether to send logs to Sentry (gets removed)
-            - dsn (str): The Sentry DSN
-            ...: Other config options for Sentry
+        server_config (Dict[str, Any]): The config dictionary returned by the API.
+            - send_logs (bool): Whether to send logs to Sentry (gets removed).
+            - dsn (str): The Sentry DSN.
+            ...: Other config options for Sentry.
+
+    Returns:
+        None.
     """
     if os.getenv("VM_NO_TELEMETRY", False):
         return
@@ -88,19 +94,27 @@ def init_sentry(server_config):
         logger.debug(f"Sentry error: {str(e)}")
 
 
-def log_performance(name=None, logger=None, force=False):
-    """Decorator to log the time it takes to run a function
+F = TypeVar("F", bound=Callable[..., Any])
+AF = TypeVar("AF", bound=Callable[..., Awaitable[Any]])
+
+
+def log_performance(
+    name: Optional[str] = None,
+    logger: Optional[logging.Logger] = None,
+    force: bool = False,
+) -> Callable[[F], F]:
+    """Decorator to log the time it takes to run a function.
 
     Args:
         name (str, optional): The name of the function. Defaults to None.
         logger (logging.Logger, optional): The logger to use. Defaults to None.
-        force (bool, optional): Whether to force logging even if env var is off
+        force (bool, optional): Whether to force logging even if env var is off.
 
     Returns:
-        function: The decorated function
+        Callable: The decorated function.
     """
 
-    def decorator(func):
+    def decorator(func: F) -> F:
         # check if log level is set to debug
         if _get_log_level() != logging.DEBUG and not force:
             return func
@@ -113,7 +127,7 @@ def log_performance(name=None, logger=None, force=False):
         if name is None:
             name = func.__name__
 
-        def wrapped(*args, **kwargs):
+        def wrapped(*args: Any, **kwargs: Any) -> Any:
             time1 = time.perf_counter()
             return_val = func(*args, **kwargs)
             time2 = time.perf_counter()
@@ -127,18 +141,13 @@ def log_performance(name=None, logger=None, force=False):
     return decorator
 
 
-async def log_performance_async(func, name=None, logger=None, force=False):
-    """Decorator to log the time it takes to run an async function
-
-    Args:
-        func (function): The function to decorate
-        name (str, optional): The name of the function. Defaults to None.
-        logger (logging.Logger, optional): The logger to use. Defaults to None.
-        force (bool, optional): Whether to force logging even if env var is off
-
-    Returns:
-        function: The decorated function
-    """
+async def log_performance_async(
+    func: AF,
+    name: Optional[str] = None,
+    logger: Optional[logging.Logger] = None,
+    force: bool = False,
+) -> AF:
+    """Async version of log_performance decorator"""
     # check if log level is set to debug
     if _get_log_level() != logging.DEBUG and not force:
         return func
@@ -149,7 +158,7 @@ async def log_performance_async(func, name=None, logger=None, force=False):
     if name is None:
         name = func.__name__
 
-    async def wrap(*args, **kwargs):
+    async def wrap(*args: Any, **kwargs: Any) -> Any:
         time1 = time.perf_counter()
         return_val = await func(*args, **kwargs)
         time2 = time.perf_counter()
@@ -161,11 +170,11 @@ async def log_performance_async(func, name=None, logger=None, force=False):
     return wrap
 
 
-def send_single_error(error: Exception):
-    """Send a single error to Sentry
+def send_single_error(error: Exception) -> None:
+    """Send a single error to Sentry.
 
     Args:
-        error (Exception): The exception to send
+        error (Exception): The exception to send.
     """
     event, hint = event_from_exception(exc_info_from_error(error))
     client = sentry_sdk.Client(__dsn, release=f"validmind-python@{__version__}")

validmind/models/foundation.py

@@ -26,9 +26,9 @@ class FoundationModel(FunctionModel):
 
     Attributes:
         predict_fn (callable): The predict function that should take a prompt as input
-          and return the result from the model
+                and return the result from the model
         prompt (Prompt): The prompt object that defines the prompt template and the
-          variables (if any)
+                variables (if any)
         name (str, optional): The name of the model. Defaults to name of the predict_fn
     """
 

validmind/models/function.py

@@ -2,6 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Any, Dict, List
+
 from validmind.vm_models.model import VMModel
 
 
@@ -18,7 +20,12 @@ class Input(dict):
     def __delitem__(self, _):
         raise TypeError("Cannot delete keys from Input")
 
-    def get_new(self):
+    def get_new(self) -> Dict[str, Any]:
+        """Get the newly added key-value pairs.
+
+        Returns:
+            Dict[str, Any]: Dictionary containing only the newly added key-value pairs.
+        """
         return {k: self[k] for k in self._new}
 
 
@@ -41,13 +48,13 @@ class FunctionModel(VMModel):
 
         self.name = self.name or self.predict_fn.__name__
 
-    def predict(self, X):
+    def predict(self, X) -> List[Any]:
         """Compute predictions for the input (X)
 
         Args:
             X (pandas.DataFrame): The input features to predict on
 
         Returns:
-            list: The predictions
+            List[Any]: The predictions
         """
         return [self.predict_fn(x) for x in X.to_dict(orient="records")]

validmind/template.py

@@ -2,7 +2,9 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from ipywidgets import HTML, Accordion, VBox
+from typing import Any, Dict, List, Optional, Type, Union
+
+from ipywidgets import HTML, Accordion, VBox, Widget
 
 from .html_templates.content_blocks import (
     failed_content_block_html,
@@ -29,8 +31,10 @@ CONTENT_TYPE_MAP = {
 
 
 def _convert_sections_to_section_tree(
-    sections, parent_id="_root_", start_section_id=None
-):
+    sections: List[Dict[str, Any]],
+    parent_id: str = "_root_",
+    start_section_id: Optional[str] = None,
+) -> List[Dict[str, Any]]:
     section_tree = []
 
     for section in sections:
@@ -49,11 +53,12 @@ def _convert_sections_to_section_tree(
 
     if start_section_id and not section_tree:
         raise ValueError(f"Section {start_section_id} not found in template")
-
-    return sorted(section_tree, key=lambda x: x.get("order", 0))
+    # sort the section tree by the order of the sections in the template (if provided)
+    # set the order to 9999 for the sections that do not have an order
+    return sorted(section_tree, key=lambda x: x.get("order", 9999))
 
 
-def _create_content_widget(content):
+def _create_content_widget(content: Dict[str, Any]) -> Widget:
     content_type = CONTENT_TYPE_MAP[content["content_type"]]
 
     if content["content_type"] not in ["metric", "test"]:
@@ -75,7 +80,9 @@ def _create_content_widget(content):
     )
 
 
-def _create_sub_section_widget(sub_sections, section_number):
+def _create_sub_section_widget(
+    sub_sections: List[Dict[str, Any]], section_number: str
+) -> Union[HTML, Accordion]:
     if not sub_sections:
         return HTML("<p>Empty Section</p>")
 
@@ -111,7 +118,7 @@ def _create_sub_section_widget(sub_sections, section_number):
     return accordion
 
 
-def _create_section_widget(tree):
+def _create_section_widget(tree: List[Dict[str, Any]]) -> Accordion:
     widget = Accordion()
     for i, section in enumerate(tree):
         sub_widget = None
@@ -139,11 +146,11 @@ def _create_section_widget(tree):
     return widget
 
 
-def preview_template(template):
-    """Preview a template in Jupyter Notebook
+def preview_template(template: str) -> None:
+    """Preview a template in Jupyter Notebook.
 
     Args:
-        template (dict): The template to preview
+        template (dict): The template to preview.
     """
     if not is_notebook():
         logger.warning("preview_template() only works in Jupyter Notebook")
@@ -154,7 +161,7 @@ def preview_template(template):
     )
 
 
-def _get_section_tests(section):
+def _get_section_tests(section: Dict[str, Any]) -> List[str]:
     """
     Get all the tests in a section and its subsections.
 
@@ -179,15 +186,15 @@ def _get_section_tests(section):
     return tests
 
 
-def _create_test_suite_section(section):
+def _create_test_suite_section(section: Dict[str, Any]) -> Dict[str, Any]:
     """Create a section object for a test suite that contains the tests in a section
-    in the template
+    in the template.
 
     Args:
-        section: a section of a template (in tree form)
+        section: A section of a template (in tree form).
 
     Returns:
-        A TestSuite section dict
+        A TestSuite section dict.
     """
     if section_tests := _get_section_tests(section):
         return {
@@ -197,16 +204,18 @@ def _create_test_suite_section(section):
         }
 
 
-def _create_template_test_suite(template, section=None):
+def _create_template_test_suite(
+    template: str, section: Optional[str] = None
+) -> Type[TestSuite]:
     """
     Create and run a test suite from a template.
 
     Args:
-        template: A valid flat template
-        section: The section of the template to run (if not provided, run all sections)
+        template: A valid flat template.
+        section: The section of the template to run. Runs all sections if not provided.
 
     Returns:
-        A dynamically-create TestSuite Class
+        A dynamically-created TestSuite Class.
     """
     section_tree = _convert_sections_to_section_tree(
         sections=template["sections"],
@@ -229,17 +238,17 @@ def _create_template_test_suite(template, section=None):
     )
 
 
-def get_template_test_suite(template, section=None):
-    """Get a TestSuite instance containing all tests in a template
+def get_template_test_suite(template: str, section: Optional[str] = None) -> TestSuite:
+    """Get a TestSuite instance containing all tests in a template.
 
     This function will collect all tests used in a template into a dynamically-created
-    TestSuite object
+    TestSuite object.
 
     Args:
         template: A valid flat template
         section: The section of the template to run (if not provided, run all sections)
 
     Returns:
-        The TestSuite instance
+        The TestSuite instance.
     """
     return _create_template_test_suite(template, section)()

validmind/test_suites/__init__.py

@@ -141,7 +141,7 @@ def list_suites(pretty: bool = True):
     return format_dataframe(pd.DataFrame(table))
 
 
-def describe_suite(test_suite_id: str, verbose=False):
+def describe_suite(test_suite_id: str, verbose: bool = False) -> pd.DataFrame:
     """
     Describes a Test Suite by ID
 
@@ -150,7 +150,7 @@ def describe_suite(test_suite_id: str, verbose=False):
         verbose: If True, describe all plans and tests in the Test Suite
 
     Returns:
-        pandas.DataFrame: A formatted table with the Test Suite description
+        pd.DataFrame: A formatted table with the Test Suite description
     """
     test_suite = get_by_id(test_suite_id)
 

validmind/tests/_store.py

@@ -5,6 +5,8 @@
 """Module for storing loaded tests and test providers"""
 
 
+from typing import Any, Callable, Optional
+
 from .test_providers import TestProvider, ValidMindTestProvider
 
 
@@ -65,19 +67,26 @@ class TestStore:
     def __init__(self):
         self.tests = {}
 
-    def get_test(self, test_id: str):
+    def get_test(self, test_id: str) -> Optional[Callable[..., Any]]:
         """Get a test by test ID
 
         Args:
             test_id (str): The test ID
 
         Returns:
-            object: The test class or function
+            Optional[Callable[..., Any]]: The test function if found, None otherwise
         """
         return self.tests.get(test_id)
 
-    def register_test(self, test_id: str, test: object = None):
-        """Register a test"""
+    def register_test(
+        self, test_id: str, test: Optional[Callable[..., Any]] = None
+    ) -> None:
+        """Register a test
+
+        Args:
+            test_id (str): The test ID
+            test (Optional[Callable[..., Any]], optional): The test function. Defaults to None.
+        """
         self.tests[test_id] = test
 
 

validmind/tests/comparison.py

@@ -146,7 +146,9 @@ def _combine_tables(results: List[TestResult]) -> List[pd.DataFrame]:
     return [_combine_single_table(results, i) for i in range(len(results[0].tables))]
 
 
-def _build_input_param_string(result: TestResult, results: List[TestResult]) -> str:
+def _build_input_param_string(
+    result: TestResult, results: List[TestResult], show_params: bool
+) -> str:
     """Build a string repr of unique inputs + params for a figure title"""
     parts = []
     unique_inputs = _get_unique_inputs(results)
@@ -162,19 +164,29 @@ def _build_input_param_string(result: TestResult, results: List[TestResult]) ->
                 input_val = _get_input_key(input_obj)
                 parts.append(f"{input_name}={input_val}")
 
-    # TODO: revisit this when we can create a value/title to show for params
-    # unique_params = _get_unique_params(results)
-    # # if theres only one unique value for a param, don't show it
-    # # however, if there is only one unique value for all params then show it as
-    # # long as there is no existing inputs in the parts list
-    # if result.params:
-    #     should_show = (
-    #         all(len(unique_params[param_name]) == 1 for param_name in unique_params)
-    #         and not parts
-    #     )
-    #     for param_name, param_value in result.params.items():
-    #         if should_show or len(unique_params[param_name]) > 1:
-    #             parts.append(f"{param_name}={param_value}")
+    # Handle params if show_params is enabled
+    if show_params and result.params:
+        unique_params = _get_unique_params(results)
+        # If there's only one unique value for a param, don't show it
+        # unless there is only one unique value for all params and no inputs shown
+        should_show = (
+            all(len(unique_params[param_name]) == 1 for param_name in unique_params)
+            and not parts
+        )
+        for param_name, param_value in result.params.items():
+            if should_show or len(unique_params[param_name]) > 1:
+                # Convert the param_value to a string representation
+                if isinstance(param_value, list):
+                    # For lists, join elements with commas
+                    str_value = ",".join(str(v) for v in param_value)
+                elif hasattr(param_value, "__str__"):
+                    # Use string representation if available
+                    str_value = str(param_value)
+                else:
+                    # Default fallback
+                    str_value = repr(param_value)
+
+                parts.append(f"{param_name}={str_value}")
 
     return ", ".join(parts)
 
@@ -207,7 +219,7 @@ def _update_figure_title(figure: Any, input_param_str: str) -> None:
         raise ValueError(f"Unsupported figure type: {type(figure)}")
 
 
-def _combine_figures(results: List[TestResult]) -> List[Any]:
+def _combine_figures(results: List[TestResult], show_params: bool) -> List[Any]:
     """Combine figures from multiple test results (gets raw figure objects, not vm Figures)"""
     combined_figures = []
 
@@ -216,7 +228,7 @@ def _combine_figures(results: List[TestResult]) -> List[Any]:
             # update the figure object in-place with the new title
             _update_figure_title(
                 figure=figure.figure,
-                input_param_str=_build_input_param_string(result, results),
+                input_param_str=_build_input_param_string(result, results, show_params),
             )
             combined_figures.append(figure)
 
@@ -279,35 +291,53 @@ def get_comparison_test_configs(
         A list of test configurations.
     """
 
-    # Convert list of dicts to dict of lists if necessary
+    # Convert list of dicts to dict of lists if necessary for input_grid
     def list_to_dict(grid_list):
         return {k: [d[k] for d in grid_list] for k in grid_list[0].keys()}
 
+    # Handle input_grid the same way as before
     if isinstance(input_grid, list):
         input_grid = list_to_dict(input_grid)
 
-    if isinstance(param_grid, list):
-        param_grid = list_to_dict(param_grid)
-
     test_configs = []
 
-    if input_grid and param_grid:
-        input_combinations = _cartesian_product(input_grid)
-        param_combinations = _cartesian_product(param_grid)
-        test_configs = [
-            {"inputs": i, "params": p}
-            for i, p in product(input_combinations, param_combinations)
-        ]
+    # Check if param_grid is a list of dictionaries
+    is_param_grid_list = isinstance(param_grid, list)
+
+    # Special handling for list-based param_grid
+    if is_param_grid_list:
+        if input_grid:
+            # Generate all combinations of input_grid and each param dictionary
+            input_combinations = _cartesian_product(input_grid)
+            test_configs = [
+                {"inputs": i, "params": p}
+                for i in input_combinations
+                for p in param_grid
+            ]
+        else:
+            # Each dictionary in param_grid is a specific test configuration
+            test_configs = [{"inputs": inputs or {}, "params": p} for p in param_grid]
+
+    # Dictionary-based param_grid
+    elif param_grid:
+        if input_grid:
+            input_combinations = _cartesian_product(input_grid)
+            param_combinations = _cartesian_product(param_grid)
+            test_configs = [
+                {"inputs": i, "params": p}
+                for i, p in product(input_combinations, param_combinations)
+            ]
+        else:
+            param_combinations = _cartesian_product(param_grid)
+            test_configs = [
+                {"inputs": inputs or {}, "params": p} for p in param_combinations
+            ]
+    # Just input_grid, no param_grid
     elif input_grid:
         input_combinations = _cartesian_product(input_grid)
         test_configs = [
             {"inputs": i, "params": params or {}} for i in input_combinations
         ]
-    elif param_grid:
-        param_combinations = _cartesian_product(param_grid)
-        test_configs = [
-            {"inputs": inputs or {}, "params": p} for p in param_combinations
-        ]
 
     return test_configs
 
@@ -333,12 +363,14 @@ def _combine_raw_data(results: List[TestResult]) -> RawData:
 
 def combine_results(
     results: List[TestResult],
+    show_params: bool,
 ) -> Tuple[List[Any], Dict[str, List[Any]], Dict[str, List[Any]]]:
     """
     Combine multiple test results into a single set of outputs.
 
     Args:
         results: A list of TestResult objects to combine.
+        show_params: Whether to show parameter values in figure titles.
 
     Returns:
         A tuple containing:
@@ -353,7 +385,7 @@ def combine_results(
     # handle tables (if any)
     combined_outputs.extend(_combine_tables(results))
     # handle figures (if any)
-    combined_outputs.extend(_combine_figures(results))
+    combined_outputs.extend(_combine_figures(results, show_params))
     # handle threshold tests (i.e. tests that have pass/fail bool status)
     if results[0].passed is not None:
         combined_outputs.append(all(result.passed for result in results))