validmind 2.5.25__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/client.py

@@ -27,7 +27,7 @@ from .template import get_template_test_suite
 from .template import preview_template as _preview_template
 from .test_suites import get_by_id as get_test_suite_by_id
 from .utils import get_dataset_info, get_model_info
-from .vm_models import TestInput, TestSuite, TestSuiteRunner
+from .vm_models import TestSuite, TestSuiteRunner
 from .vm_models.dataset import DataFrameDataset, PolarsDataset, TorchDataset, VMDataset
 from .vm_models.model import (
     ModelAttributes,
@@ -100,7 +100,6 @@ def init_dataset(
 
     # Instantiate supported dataset types here
     if isinstance(dataset, pd.DataFrame):
-        logger.info("Pandas dataset detected. Initializing VM Dataset instance...")
         vm_dataset = DataFrameDataset(
             input_id=input_id,
             raw_dataset=dataset,
@@ -113,7 +112,6 @@ def init_dataset(
             date_time_index=date_time_index,
         )
     elif isinstance(dataset, pl.DataFrame):
-        logger.info("Polars dataset detected. Initializing VM Dataset instance...")
         vm_dataset = PolarsDataset(
             input_id=input_id,
             raw_dataset=dataset,
@@ -126,7 +124,6 @@ def init_dataset(
             date_time_index=date_time_index,
         )
     elif dataset_class == "ndarray":
-        logger.info("Numpy ndarray detected. Initializing VM Dataset instance...")
         vm_dataset = VMDataset(
             input_id=input_id,
             raw_dataset=dataset,
@@ -143,7 +140,6 @@ def init_dataset(
             date_time_index=date_time_index,
         )
     elif dataset_class == "TensorDataset":
-        logger.info("Torch TensorDataset detected. Initializing VM Dataset instance...")
         vm_dataset = TorchDataset(
             input_id=input_id,
             raw_dataset=dataset,
@@ -277,6 +273,7 @@ def init_model(
 
 def init_r_model(
     model_path: str,
+    input_id: str = "model",
 ) -> VMModel:
     """
     Initializes a VM Model for an R model
@@ -325,6 +322,7 @@ def init_r_model(
     vm_model = RModel(
         r=r,
         model=model,
+        input_id=input_id,
     )
 
     return vm_model
@@ -409,7 +407,7 @@ def run_test_suite(
 
     TestSuiteRunner(
         suite=suite,
-        input=TestInput({**kwargs, **(inputs or {})}),
+        inputs={**kwargs, **(inputs or {})},
         config=config or {},
     ).run(fail_fast=fail_fast, send=send)
 
@@ -512,7 +510,7 @@ def _run_documentation_section(
 
     TestSuiteRunner(
         suite=test_suite,
-        input=TestInput({**kwargs, **(inputs or {})}),
+        inputs={**kwargs, **(inputs or {})},
         config=config,
     ).run(send=send, fail_fast=fail_fast)
 

validmind/client_config.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-Central class to track configuration of the library
+Central class to track configuration of the ValidMind Library
 client against the ValidMind API
 """
 

validmind/datasets/__init__.py

@@ -3,5 +3,5 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-Example datasets that can be used with the library.
+Example datasets that can be used with the ValidMind Library.
 """

validmind/datasets/classification/customer_churn.py

@@ -86,16 +86,18 @@ def get_demo_test_config(test_suite=None):
     for _, test_config in default_config.items():
         if "model" in test_config["inputs"]:
             test_config["inputs"]["model"] = "model"
+        if "models" in test_config["inputs"]:
+            test_config["inputs"]["models"] = ["model"]
+        if "dataset" in test_config["inputs"]:
+            if "model" in test_config["inputs"] or "models" in test_config["inputs"]:
+                test_config["inputs"]["dataset"] = "test_dataset"
+            else:
+                test_config["inputs"]["dataset"] = "raw_dataset"
         if "datasets" in test_config["inputs"]:
             test_config["inputs"]["datasets"] = [
                 "train_dataset",
                 "test_dataset",
             ]
-        if "dataset" in test_config["inputs"]:
-            if "model" in test_config["inputs"]:
-                test_config["inputs"]["dataset"] = "test_dataset"
-            else:
-                test_config["inputs"]["dataset"] = "raw_dataset"
 
     # ClassifierPerformance is a special case since we run an in-sample and out-of-sample
     # test with two different datasets: train_dataset and test_dataset

validmind/datasets/nlp/__init__.py

@@ -3,10 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-Example datasets that can be used with the library.
+Example datasets that can be used with the ValidMind Library.
 """
 
 __all__ = [
     "cnn_dailymail",
-    "twitter_covide_19",
+    "twitter_covid_19",
 ]

validmind/errors.py

@@ -3,13 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-This module contains all the custom errors that are used in the library.
+This module contains all the custom errors that are used in the ValidMind Library.
 
 The following base errors are defined for others:
 - BaseError
 - APIRequestError
 """
+
 import json
+from typing import Optional
 
 
 class BaseError(Exception):
@@ -79,14 +81,6 @@ class InvalidContentIdPrefixError(APIRequestError):
     """
 
 
-class InvalidFigureForObjectError(BaseError):
-    """
-    When a figure was constructed with an unsupported for_object value.
-    """
-
-    pass
-
-
 class InvalidMetricResultsError(APIRequestError):
     """
     When an invalid metric results object is sent to the API.
@@ -164,7 +158,9 @@ class LoadTestError(BaseError):
     Exception raised when an error occurs while loading a test
     """
 
-    pass
+    def __init__(self, message: str, original_error: Optional[Exception] = None):
+        super().__init__(message)
+        self.original_error = original_error
 
 
 class MismatchingClassLabelsError(BaseError):

validmind/html_templates/content_blocks.py

@@ -14,7 +14,24 @@ non_test_content_block_html = """
 test_content_block_html = """
 <div>
   <h2>{title}</h2>
-  {description}
+  <div style="border: 1px solid #ddd; border-radius: 4px; padding: 10px; margin: 10px 0;">
+    {description}
+  </div>
+</div>
+
+<h4 class="vm_required_context">
+  Required Inputs: <span style="font-size: 13px"><i>{required_inputs}</i></span>
+</h4>
+
+<div style="display: {table_display};">
+  <h4>Parameters:</h4>
+  <table class="vm_params_table" style="display: {table_display};">
+      <tr>
+          <th>Parameter</th>
+          <th>Default Value</th>
+      </tr>
+      {params_table}
+  </table>
 </div>
 
 <div class="unset">
@@ -44,21 +61,6 @@ result = vm.tests.run_test(
 # To see the result of the test, ensure that you have called `vm.init()` and then run:
 result.log()</code>
     </pre>
-
-    <h4 class="vm_required_context">
-      Required Inputs: <span style="font-size: 13px"><i>{required_inputs}</i></span>
-    </h4>
-
-    <div style="display: {table_display};">
-      <h4>Parameters:</h4>
-      <table class="vm_params_table" style="display: {table_display};">
-          <tr>
-              <th>Parameter</th>
-              <th>Default Value</th>
-          </tr>
-          {params_table}
-      </table>
-    </div>
   </div>
 </div>
 

validmind/logging.py

@@ -88,11 +88,10 @@ def init_sentry(server_config):
         logger.debug(f"Sentry error: {str(e)}")
 
 
-def log_performance(func, name=None, logger=None, force=False):
+def log_performance(name=None, logger=None, force=False):
     """Decorator to log the time it takes to run a function
 
     Args:
-        func (function): The function to decorate
         name (str, optional): The name of the function. Defaults to None.
         logger (logging.Logger, optional): The logger to use. Defaults to None.
         force (bool, optional): Whether to force logging even if env var is off
@@ -100,26 +99,32 @@ def log_performance(func, name=None, logger=None, force=False):
     Returns:
         function: The decorated function
     """
-    # check if log level is set to debug
-    if _get_log_level() != logging.DEBUG and not force:
-        return func
 
-    if logger is None:
-        logger = get_logger()
+    def decorator(func):
+        # check if log level is set to debug
+        if _get_log_level() != logging.DEBUG and not force:
+            return func
 
-    if name is None:
-        name = func.__name__
+        nonlocal logger
+        if logger is None:
+            logger = get_logger()
 
-    def wrap(*args, **kwargs):
-        time1 = time.perf_counter()
-        return_val = func(*args, **kwargs)
-        time2 = time.perf_counter()
+        nonlocal name
+        if name is None:
+            name = func.__name__
 
-        logger.debug("%s function took %0.3f ms" % (name, (time2 - time1) * 1000.0))
+        def wrapped(*args, **kwargs):
+            time1 = time.perf_counter()
+            return_val = func(*args, **kwargs)
+            time2 = time.perf_counter()
 
-        return return_val
+            logger.debug("%s function took %0.3f ms" % (name, (time2 - time1) * 1000.0))
 
-    return wrap
+            return return_val
+
+        return wrapped
+
+    return decorator
 
 
 async def log_performance_async(func, name=None, logger=None, force=False):

validmind/tests/__init__.py

@@ -6,15 +6,39 @@
 
 from ..errors import LoadTestError
 from ..logging import get_logger
-from .decorator import metric, tags, tasks, test
-from .load import describe_test, list_tests, load_test
-from .metadata import list_tags, list_tasks, list_tasks_and_tags
+from ._store import test_provider_store
+from .decorator import tags, tasks, test
+from .load import (
+    describe_test,
+    list_tags,
+    list_tasks,
+    list_tasks_and_tags,
+    list_tests,
+    load_test,
+)
 from .run import run_test
-from .test_providers import LocalTestProvider, TestProvider, register_test_provider
+from .test_providers import LocalTestProvider, TestProvider
 
 logger = get_logger(__name__)
 
 
+# public-facing function for registering test providers
+def register_test_provider(namespace: str, test_provider: TestProvider) -> None:
+    """Register an external test provider
+
+    Args:
+        namespace (str): The namespace of the test provider
+        test_provider (TestProvider): The test provider
+    """
+    if not hasattr(test_provider, "list_tests"):
+        raise ValueError("test_provider must implement `list_tests` method")
+
+    if not hasattr(test_provider, "load_test"):
+        raise ValueError("test_provider must implement `load_test` method")
+
+    test_provider_store.register_test_provider(namespace, test_provider)
+
+
 __all__ = [
     "data_validation",
     "model_validation",
@@ -33,7 +57,6 @@ __all__ = [
     "list_tasks_and_tags",
     # Decorators for functional metrics
     "test",
-    "metric",  # DEPRECATED
     "tags",
     "tasks",
 ]