validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/vm_models/result/utils.py

@@ -0,0 +1,160 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import os
+from typing import TYPE_CHECKING, Dict, List, Union
+
+import pandas as pd
+from ipywidgets import HTML, GridBox, Layout
+from jinja2 import Template
+
+from ... import api_client
+from ...logging import get_logger
+from ..dataset import VMDataset
+from ..figure import Figure
+from ..input import VMInput
+
+if TYPE_CHECKING:
+    from .result import ResultTable
+
+
+AI_REVISION_NAME = "Generated by ValidMind AI"
+DEFAULT_REVISION_NAME = "Default Description"
+
+logger = get_logger(__name__)
+
+_result_template = None
+
+
+def get_result_template():
+    """Get the jinja html template for rendering test results"""
+    global _result_template
+
+    if _result_template is None:
+        with open(os.path.join(os.path.dirname(__file__), "result.jinja")) as f:
+            _result_template = Template(f.read())
+
+    return _result_template
+
+
+async def update_metadata(content_id: str, text: str, _json: Union[Dict, List] = None):
+    """Create or Update a Metadata Object"""
+    parts = content_id.split("::")
+    content_id = parts[0]
+    revision_name = parts[1] if len(parts) > 1 else None
+
+    # we always want composite metric definitions to be updated
+    should_update = content_id.startswith("composite_metric_def:")
+
+    # if we are updating a metric or test description, we check if the text
+    # has changed from the last time it was logged, and only update if it has
+    if content_id.split(":", 1)[0] in ["metric_description", "test_description"]:
+        try:
+            md = await api_client.aget_metadata(content_id)
+            # if there is an existing description, only update it if the new one
+            # is different and is an AI-generated description
+            should_update = (
+                md["text"] != text if revision_name == AI_REVISION_NAME else False
+            )
+            logger.debug(f"Check if description has changed: {should_update}")
+        except Exception:
+            # if exception, assume its not created yet TODO: don't catch all
+            should_update = True
+
+    if should_update:
+        if revision_name:
+            content_id = f"{content_id}::{revision_name}"
+
+        logger.debug(f"Updating metadata for `{content_id}`")
+
+        await api_client.alog_metadata(content_id, text, _json)
+
+
+def check_for_sensitive_data(data: pd.DataFrame, inputs: List[VMInput]):
+    """Check if a table contains raw data from input datasets"""
+    dataset_columns = {
+        col: len(input_obj.df)
+        for input_obj in inputs
+        if isinstance(input_obj, VMDataset)
+        for col in input_obj.columns
+    }
+
+    table_columns = {col: len(data) for col in data.columns}
+
+    offending_columns = [
+        col
+        for col in table_columns
+        if col in dataset_columns and table_columns[col] == dataset_columns[col]
+    ]
+
+    if offending_columns:
+        raise ValueError(
+            f"Raw input data found in table, pass `unsafe=True` "
+            f"or remove the offending columns: {offending_columns}"
+        )
+
+
+def tables_to_widgets(tables: List["ResultTable"]):
+    """Convert summary (list of json tables) into a list of ipywidgets"""
+    widgets = [
+        HTML("<h3>Tables</h3>"),
+    ]
+
+    for table in tables:
+        html = ""
+        if table.title:
+            html += f"<h4>{table.title}</h4>"
+
+        html += (
+            table.data.reset_index(drop=True)
+            .style.format(precision=4)
+            .hide(axis="index")
+            .set_table_styles(
+                [
+                    {
+                        "selector": "",
+                        "props": [("width", "100%")],
+                    },
+                    {
+                        "selector": "th",
+                        "props": [("text-align", "left")],
+                    },
+                    {
+                        "selector": "tbody tr:nth-child(even)",
+                        "props": [("background-color", "#FFFFFF")],
+                    },
+                    {
+                        "selector": "tbody tr:nth-child(odd)",
+                        "props": [("background-color", "#F5F5F5")],
+                    },
+                    {
+                        "selector": "td, th",
+                        "props": [
+                            ("padding-left", "5px"),
+                            ("padding-right", "5px"),
+                        ],
+                    },
+                ]
+            )
+            .set_properties(**{"text-align": "left"})
+            .to_html(escape=False)
+        )
+
+        widgets.append(HTML(html))
+
+    return widgets
+
+
+def figures_to_widgets(figures: List[Figure]) -> list:
+    """Plot figures to a ipywidgets GridBox"""
+    num_columns = 2 if len(figures) > 1 else 1
+
+    plot_widgets = GridBox(
+        [figure.to_widget() for figure in figures],
+        layout=Layout(
+            grid_template_columns=f"repeat({num_columns}, 1fr)",
+        ),
+    )
+
+    return [HTML("<h3>Figures</h3>"), plot_widgets]

validmind/vm_models/test_suite/runner.py

@@ -9,7 +9,6 @@ from IPython.display import display
 
 from ...logging import get_logger
 from ...utils import is_notebook, run_async, run_async_check
-from ..test_context import TestContext, TestInput
 from .summary import TestSuiteSummary
 from .test_suite import TestSuite
 
@@ -22,8 +21,6 @@ class TestSuiteRunner:
     """
 
     suite: TestSuite = None
-    context: TestContext = None
-    input: TestInput = None
     config: dict = None
 
     _test_configs: dict = None
@@ -32,67 +29,33 @@ class TestSuiteRunner:
     pbar_description: widgets.Label = None
     pbar_box: widgets.HBox = None
 
-    def __init__(self, suite: TestSuite, input: TestInput, config: dict = None):
+    def __init__(self, suite: TestSuite, config: dict = None, inputs: dict = None):
         self.suite = suite
-        self.input = input
         self.config = config or {}
 
-        self.context = TestContext()
+        self._load_config(inputs)
 
-        self._load_config()
-        self._init_tests()
-
-    def _load_config(self):
+    def _load_config(self, inputs: dict = None):
         """Splits the config into a global config and test configs"""
-        self._test_configs = {}
+        self._test_configs = {
+            test.test_id: {"inputs": inputs or {}} for test in self.suite.get_tests()
+        }
 
         for key, value in self.config.items():
-            test_ids = [test.test_id for test in self.suite.get_tests()]
-
             # If the key does not exist in the test suite, we need to
             # inform the user the config is probably wrong but we will
             # keep running all tests
-            if key not in test_ids:
+            if key not in self._test_configs:
                 logger.warning(
                     f"Config key '{key}' does not match a test_id in the template."
                     "\n\tEnsure you registered a content block with the correct content_id in the template"
                     "\n\tThe configuration for this test will be ignored."
                 )
-            else:
-                self._test_configs[key] = value
+                continue
 
-    def _init_tests(self):
-        """
-        Loads the tests in a test suite
-        """
-        for section in self.suite.sections:
-            for test in section.tests:
-                # use local inputs from config if provided
-                test_configs = self._test_configs.get(test.test_id, {})
-                inputs = self.input
-                if (
-                    test.test_id in self.config
-                    and "inputs" in self.config[test.test_id]
-                ):
-                    inputs = TestInput(self.config[test.test_id]["inputs"])
-                    test_configs = {
-                        key: value
-                        for key, value in test_configs.items()
-                        if key != "inputs"
-                    }
-                    test_configs = test_configs.get("params", {})
-                else:
-                    if (test_configs) and ("params" not in test_configs):
-                        # [DEPRECATED] This is the old way of setting test parameters
-                        msg = (
-                            "Setting test parameters directly in the 'config' parameter"
-                            " of the run_documentation_tests() method is deprecated. "
-                            "Instead, use the new format of the config: "
-                            'config = {"test_id": {"params": {...}, "inputs": {...}}}'
-                        )
-                        logger.warning(msg)
-
-                test.load(inputs=inputs, context=self.context, config=test_configs)
+            # override the global config (inputs) with the test-specific config
+            # TODO: better configuration would make for a better DX
+            self._test_configs[key] = value
 
     def _start_progress_bar(self, send: bool = True):
         """
@@ -176,12 +139,11 @@ class TestSuiteRunner:
 
         for section in self.suite.sections:
             for test in section.tests:
-                if test._test_class is None:
-                    self.pbar.value += 1
-                    continue
-
-                self.pbar_description.value = f"Running {test.test_type}: {test.name}"
-                test.run(fail_fast=fail_fast)
+                self.pbar_description.value = f"Running {test.name}"
+                test.run(
+                    fail_fast=fail_fast,
+                    config=self._test_configs.get(test.test_id, {}),
+                )
                 self.pbar.value += 1
 
         if send:

validmind/vm_models/test_suite/summary.py

@@ -9,7 +9,7 @@ import ipywidgets as widgets
 
 from ...logging import get_logger
 from ...utils import display, md_to_html
-from ..test.result_wrapper import FailedResultWrapper
+from ..result import ErrorResult
 from .test_suite import TestSuiteSection, TestSuiteTest
 
 logger = get_logger(__name__)
@@ -52,7 +52,7 @@ class TestSuiteSectionSummary:
             children.append(test.result.to_widget())
             titles.append(
                 f"❌ {test.result.name}: {test.name} ({test.test_id})"
-                if isinstance(test.result, FailedResultWrapper)
+                if isinstance(test.result, ErrorResult)
                 else f"{test.result.name}: {test.name} ({test.test_id})"
             )
 
@@ -96,7 +96,7 @@ class TestSuiteSummary:
         from ...api_client import get_api_host, get_api_model
 
         ui_host = get_api_host().replace("/api/v1/tracking", "").replace("api", "app")
-        link = f"{ui_host}/projects/{get_api_model()}/project-overview"
+        link = f"{ui_host}model-inventory/{get_api_model()}"
         results_link = f"""
         <h3>
             Check out the updated documentation in your

validmind/vm_models/test_suite/test.py

@@ -2,14 +2,14 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from ...errors import should_raise_on_fail_fast
+from typing import Any, Dict, Union
+
+from ...errors import LoadTestError, should_raise_on_fail_fast
 from ...logging import get_logger, log_performance
-from ...tests import LoadTestError
-from ...tests import load_test as load_test_class
+from ...tests.load import load_test
+from ...tests.run import run_test
 from ...utils import test_id_to_name
-from ..test.result_wrapper import FailedResultWrapper, ResultWrapper
-from ..test.test import Test
-from ..test_context import TestContext, TestInput
+from ..result import ErrorResult, Result, TestResult
 
 logger = get_logger(__name__)
 
@@ -21,12 +21,11 @@ class TestSuiteTest:
 
     test_id: str
     output_template: str = None
-    name: str = None
-
-    _test_class: Test = None
-    _test_instance: Test = None
+    name: Union[str, None] = None
+    description: Union[Dict[str, Any], None] = None
+    result: Union[Result, None] = None
 
-    result: object = None
+    _load_failed: bool = False
 
     def __init__(self, test_id_or_obj):
         """Load the test class from the test id
@@ -42,69 +41,46 @@ class TestSuiteTest:
 
         self.name = test_id_to_name(self.test_id)
 
+    def get_default_config(self):
+        """Returns the default configuration for the test"""
         try:
-            self._test_class = load_test_class(self.test_id)
+            test_func = load_test(self.test_id)
         except LoadTestError as e:
-            self.result = FailedResultWrapper(
-                error=e,
-                message=f"Failed to load test '{self.test_id}'",
-                result_id=self.test_id,
-            )
-        except Exception as e:
-            # The test suite runner will appropriately ignore this error
-            # since _test_class is None
             logger.error(f"Failed to load test '{self.test_id}': {e}")
 
-    @property
-    def test_type(self):
-        return self._test_class.test_type
-
-    def get_default_params(self):
-        """Returns the default params for the test"""
-        if not self._test_class:
-            return {}
-
-        return self._test_class.default_params
-
-    def load(self, inputs: TestInput, context: TestContext, config: dict = None):
-        """Load an instance of the test class"""
-        if not self._test_class:
-            return
-
-        try:
-            self._test_instance = self._test_class(
-                test_id=self.test_id,
-                context=context,
-                inputs=inputs,
-                params=config,
-                output_template=self.output_template,
-            )
-        except Exception as e:
-            logger.error(
-                f"Failed to load test '{self.test_id}': "
-                f"({e.__class__.__name__}) {e}"
-            )
-            self.result = FailedResultWrapper(
+            self._load_failed = True
+            self.result = ErrorResult(
                 error=e,
                 message=f"Failed to load test '{self.name}'",
                 result_id=self.test_id,
             )
 
-    def run(self, fail_fast: bool = False):
+            return None
+
+        config = {
+            # we use the input name ('dataset', 'model') as the key and the value
+            "inputs": {k: k for k in test_func.inputs},
+            "params": {k: v.get("default") for k, v in test_func.params.items()},
+        }
+
+        return config
+
+    def run(self, fail_fast: bool = False, config: dict = None):
         """Run the test"""
-        if not self._test_instance:
-            # test failed to load and we have already logged the error
+        if self._load_failed:
             return
 
         try:
-            self._test_instance.validate_inputs()
-
             # run the test and log the performance if LOG_LEVEL is set to DEBUG
-            log_performance(
-                func=self._test_instance.run,
-                name=self.test_id,
-                logger=logger,
-            )()  # this is a decorator so we need to call it
+            @log_performance(name=self.test_id, logger=logger)
+            def run_test_with_logging():
+                return run_test(
+                    self.test_id,
+                    **(config or {}),
+                    show=False,
+                )
+
+            self.result = run_test_with_logging()
 
         except Exception as e:
             if fail_fast and should_raise_on_fail_fast(e):
@@ -113,40 +89,30 @@ class TestSuiteTest:
             logger.error(
                 f"Failed to run test '{self.test_id}': " f"({e.__class__.__name__}) {e}"
             )
-            self.result = FailedResultWrapper(
-                name=f"Failed {self._test_instance.test_type}",
+            self.result = ErrorResult(
                 error=e,
                 message=f"Failed to run '{self.name}'",
                 result_id=self.test_id,
             )
 
-            return
-
-        if self._test_instance.result is None:
-            self.result = FailedResultWrapper(
-                name=f"Failed {self._test_instance.test_type}",
+        if self.result is None:
+            self.result = ErrorResult(
                 error=None,
                 message=f"'{self.name}' did not return a result",
                 result_id=self.test_id,
             )
 
-            return
-
-        if not isinstance(self._test_instance.result, ResultWrapper):
-            self.result = FailedResultWrapper(
-                name=f"Failed {self._test_instance.test_type}",
+        if not isinstance(self.result, Result):
+            self.result = ErrorResult(
                 error=None,
                 message=f"{self.name} returned an invalid result: {self._test_instance.result}",
                 result_id=self.test_id,
             )
 
-            return
-
-        self.result = self._test_instance.result
-
     async def log_async(self):
         """Log the result for this test to ValidMind"""
         if not self.result:
             raise ValueError("Cannot log test result before running the test")
 
-        await self.result.log_async()
+        if isinstance(self.result, TestResult):
+            return await self.result.log_async()

validmind/vm_models/test_suite/test_suite.py

@@ -48,48 +48,16 @@ class TestSuiteSection:
     section_id: str = None
     description: Optional[str] = None
 
-    def get_required_inputs_for_test(self, test: TestSuiteTest) -> List[str]:
-        """
-        Returns the required inputs for a specific test. Returns an input
-        dictionary that can be passed directly to run_test() or run_documentation_test()
-
-        Args:
-            test (TestSuiteTest): The test to get the required inputs for
-
-        Returns:
-            dict: A dictionary of required inputs
-        """
-        test_class = test._test_class
-        inputs_dict = {}
-        if (
-            not hasattr(test_class, "required_inputs")
-            or test_class.required_inputs is None
-        ):
-            return inputs_dict
-
-        for input_name in test_class.required_inputs:
-            # This required input is not valid but the behavior in this function
-            # is consistent with required_inputs as defined in the test class so
-            # we will ignore it for now
-            #
-            # if input_name == "model.train_ds" or input_name == "model.test_ds":
-            #     continue
-
-            # Assign None to the input to indicate that it is required
-            inputs_dict[input_name] = None
-
-        return inputs_dict
-
     def get_default_config(self):
         """Returns the default configuration for the test suite section"""
         # TODO: configuration across sections/tests needs more work
         section_default_config = {}
 
         for test in self.tests:
-            section_default_config[test.test_id] = {
-                "inputs": self.get_required_inputs_for_test(test),
-                "params": test.get_default_params() or {},
-            }
+            default_config = test.get_default_config()
+
+            if default_config:
+                section_default_config[test.test_id] = default_config
 
         return section_default_config
 
@@ -162,13 +130,13 @@ class TestSuite:
         return self.suite_id.title().replace("_", " ")
 
     def get_tests(self) -> List[str]:
-        """Get all test IDs from all sections"""
-        test_ids = []
+        """Get all test suite test objects from all sections"""
+        tests = []
 
         for section in self.sections:
-            test_ids.extend(section.tests)
+            tests.extend(section.tests)
 
-        return test_ids
+        return tests
 
     def num_tests(self) -> int:
         """Returns the total number of tests in the test suite"""