validmind 2.0.1__tar.gz → 2.1.0__tar.gz

{validmind-2.0.1 → validmind-2.1.0}/PKG-INFO

@@ -1,16 +1,17 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.0.1
+Version: 2.1.0
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez
 Author-email: andres@validmind.ai
-Requires-Python: >=3.8,<3.11
+Requires-Python: >=3.8,<3.12
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: all
 Provides-Extra: llm
 Provides-Extra: pytorch
@@ -21,36 +22,35 @@ Requires-Dist: arch (>=5.4.0,<6.0.0)
 Requires-Dist: bert-score (>=0.3.13,<0.4.0)
 Requires-Dist: catboost (>=1.2,<2.0)
 Requires-Dist: click (>=8.0.4,<9.0.0)
-Requires-Dist: datasets (>=2.14.5,<3.0.0) ; extra == "all" or extra == "llm"
-Requires-Dist: dython (>=0.7.1,<0.8.0)
 Requires-Dist: evaluate (>=0.4.0,<0.5.0)
-Requires-Dist: ipython (==7.34.0)
 Requires-Dist: ipywidgets (>=8.0.6,<9.0.0)
-Requires-Dist: jupyter (>=1.0.0,<2.0.0)
-Requires-Dist: kaleido (==0.2.1)
+Requires-Dist: kaleido (>=0.2.1,<0.3.0,!=0.2.1.post1)
 Requires-Dist: langdetect (>=1.0.9,<2.0.0)
 Requires-Dist: levenshtein (>=0.21.1,<0.22.0) ; extra == "all" or extra == "llm"
-Requires-Dist: markdown (>=3.4.3,<4.0.0)
-Requires-Dist: matplotlib (<3.8)
+Requires-Dist: llvmlite (>=0.42.0) ; python_version >= "3.12"
+Requires-Dist: llvmlite ; python_version >= "3.8" and python_full_version <= "3.11.0"
+Requires-Dist: matplotlib (<3.8.3)
+Requires-Dist: mistune (>=3.0.2,<4.0.0)
 Requires-Dist: nltk (>=3.8.1,<4.0.0)
-Requires-Dist: numpy (>=1.23.3,<2.0.0)
+Requires-Dist: numba (<0.59.0)
+Requires-Dist: numpy (>=1.23.3,<2.0.0) ; python_version >= "3.8" and python_full_version <= "3.11.0"
+Requires-Dist: numpy (>=1.26.0) ; python_version >= "3.12"
 Requires-Dist: openai (>=1.3.7,<2.0.0) ; extra == "all" or extra == "llm"
-Requires-Dist: pandas (==1.5.3)
-Requires-Dist: pandas-profiling (>=3.6.6,<4.0.0)
-Requires-Dist: pdoc (>=13.1.1,<14.0.0)
+Requires-Dist: pandas (>=1.1,<2)
 Requires-Dist: plotly (>=5.14.1,<6.0.0)
 Requires-Dist: plotly-express (>=0.4.1,<0.5.0)
+Requires-Dist: polars (>=0.20.15,<0.21.0)
 Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"
-Requires-Dist: pydantic (>=1.9.1,<2.0.0)
 Requires-Dist: pypmml (>=0.9.17,<0.10.0)
 Requires-Dist: python-dotenv (>=0.20.0,<0.21.0)
-Requires-Dist: requests (>=2.27.1,<3.0.0)
 Requires-Dist: rouge (>=1.0.1,<2.0.0)
 Requires-Dist: rpy2 (>=3.5.10,<4.0.0) ; extra == "all" or extra == "r-support"
 Requires-Dist: scikit-learn (>=1.0.2,<2.0.0)
+Requires-Dist: scipy (>=1.12) ; python_version >= "3.12"
+Requires-Dist: scipy ; python_version >= "3.8" and python_full_version <= "3.11.0"
 Requires-Dist: scorecardpy (>=0.1.9.6,<0.2.0.0)
 Requires-Dist: seaborn (>=0.11.2,<0.12.0)
-Requires-Dist: sentencepiece (>=0.1.99,<0.2.0)
+Requires-Dist: selfcheckgpt (>=0.1.7,<0.2.0)
 Requires-Dist: sentry-sdk (>=1.24.0,<2.0.0)
 Requires-Dist: shap (>=0.42.0,<0.43.0)
 Requires-Dist: statsmodels (>=0.13.5,<0.14.0)
@@ -60,8 +60,8 @@ Requires-Dist: torch (>=1.10.0) ; extra == "all" or extra == "llm" or extra == "
 Requires-Dist: torchmetrics (>=1.1.1,<2.0.0) ; extra == "all" or extra == "llm"
 Requires-Dist: tqdm (>=4.64.0,<5.0.0)
 Requires-Dist: transformers (>=4.32.0,<5.0.0) ; extra == "all" or extra == "llm" or extra == "transformers"
-Requires-Dist: wget (>=3.2,<4.0)
-Requires-Dist: xgboost (>=1.5.2,<2.0.0)
+Requires-Dist: xgboost (>=1.5.2,<3)
+Requires-Dist: ydata-profiling (>=4.7.0,<5.0.0)
 Description-Content-Type: text/markdown
 
 # ValidMind Developer Framework

{validmind-2.0.1 → validmind-2.1.0}/pyproject.toml

@@ -10,7 +10,7 @@ description = "ValidMind Developer Framework"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.0.1"
+version = "2.1.0"
 
 [tool.poetry.dependencies]
 aiohttp = {extras = ["speedups"], version = "^3.8.4"}
@@ -18,37 +18,42 @@ arch = "^5.4.0"
 bert-score = "^0.3.13"
 catboost = "^1.2"
 click = "^8.0.4"
-datasets = {version = "^2.14.5", optional = true}
-dython = "^0.7.1"
 evaluate = "^0.4.0"
-ipython = "7.34.0"
 ipywidgets = "^8.0.6"
-jupyter = "^1.0.0"
-kaleido = "0.2.1"
+kaleido = "^0.2.1,!=0.2.1.post1"
 langdetect = "^1.0.9"
 levenshtein = {version = "^0.21.1", optional = true}
-markdown = "^3.4.3"
-matplotlib = "<3.8"
+llvmlite = [
+  {version = "*", python = ">=3.8,<=3.11"},
+  {version = ">=0.42.0", python = ">=3.12"},
+]
+matplotlib = "<3.8.3"
+mistune = "^3.0.2"
 nltk = "^3.8.1"
-numpy = "^1.23.3"
+numba = "<0.59.0" # TODO: https://github.com/validmind/developer-framework/pull/28
+numpy = [
+  {version = "^1.23.3", python = ">=3.8,<=3.11"},
+  {version = ">=1.26.0", python = ">=3.12"},
+]
 openai = {version = "^1.3.7", optional = true}
-pandas = "1.5.3"
-pandas-profiling = "^3.6.6"
-pdoc = "^13.1.1"
+pandas = ">=1.1,<2" # TODO small issues blocking 2.0 but it would speed things up to use arrow backend
 plotly = "^5.14.1"
 plotly-express = "^0.4.1"
+polars = "^0.20.15"
 pycocoevalcap = {version = "^1.2", optional = true}
-pydantic = "^1.9.1"
 pypmml = "^0.9.17"
-python = ">=3.8,<3.11"
+python = ">=3.8,<3.12"
 python-dotenv = "^0.20.0"
-requests = "^2.27.1"
 rouge = "^1.0.1"
 rpy2 = {version = "^3.5.10", optional = true}
 scikit-learn = "^1.0.2"
+scipy = [
+  {version = "*", python = ">=3.8,<=3.11"},
+  {version = ">=1.12", python = ">=3.12"},
+]
 scorecardpy = "^0.1.9.6"
 seaborn = "^0.11.2"
-sentencepiece = "^0.1.99"
+selfcheckgpt = "^0.1.7"
 sentry-sdk = "^1.24.0"
 shap = "^0.42.0"
 statsmodels = "^0.13.5"
@@ -57,9 +62,9 @@ textstat = "^0.7.3"
 torch = {version = ">=1.10.0", optional = true}
 torchmetrics = {version = "^1.1.1", optional = true}
 tqdm = "^4.64.0"
-transformers = "^4.32.0"
-wget = "^3.2"
-xgboost = "^1.5.2"
+transformers = {version = "^4.32.0", optional = true}
+xgboost = ">=1.5.2,<3"
+ydata-profiling = "^4.7.0"
 
 [tool.poetry.group.dev.dependencies]
 black = "^22.1.0"
@@ -69,9 +74,10 @@ flake8 = "^4.0.1"
 gradio = "^3.43.2"
 ipykernel = "^6.22.0"
 isort = "^5.12.0"
-jupyterlab = "^3.6.3"
+jupyter = "^1.0.0"
 openai = "^1.3.7"
 papermill = "^2.4.0"
+pdoc = "^14.4.0"
 pre-commit = "^3.3.3"
 pytest = "^5.2"
 sphinx = "^6.1.3"
@@ -88,7 +94,6 @@ all = [
   "pycocoevalcap",
   "torchmetrics",
   "levenshtein",
-  "datasets",
 ]
 llm = [
   "torch",
@@ -97,7 +102,6 @@ llm = [
   "pycocoevalcap",
   "torchmetrics",
   "levenshtein",
-  "datasets",
 ]
 pytorch = ["torch"]
 r-support = ["rpy2"]
@@ -110,3 +114,6 @@ requires = ["poetry-core>=1.0.0"]
 [tool.isort]
 known_first_party = "validmind"
 profile = "black"
+
+[tool.poetry.scripts]
+vm-create-new-test = "scripts.create_new_test:generate_test"

{validmind-2.0.1 → validmind-2.1.0}/validmind/__init__.py

@@ -50,7 +50,7 @@ from .__version__ import __version__  # noqa: E402
 from .api_client import init
 from .api_client import log_figure as _log_figure_async
 from .api_client import log_metrics as _log_metrics_async
-from .api_client import log_test_results
+from .api_client import log_test_results, reload
 from .client import (  # noqa: E402
     get_test_suite,
     init_dataset,
@@ -60,7 +60,7 @@ from .client import (  # noqa: E402
     run_documentation_tests,
     run_test_suite,
 )
-from .unit_metrics import run_metric
+from .tests.decorator import metric, tags, tasks
 from .utils import run_async  # noqa: E402
 
 
@@ -105,9 +105,13 @@ __all__ = [  # noqa
     "init_dataset",
     "init_model",
     "init_r_model",
+    "metric",
     "preview_template",
+    "reload",
     "run_documentation_tests",
     "run_test_suite",
+    "tags",
+    "tasks",
     "tests",
     "test_suites",
     "vm_models",
@@ -116,5 +120,4 @@ __all__ = [  # noqa
     "log_figure",
     "log_metrics",
     "log_test_results",
-    "run_metric",
 ]

validmind-2.1.0/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.1.0"

validmind-2.1.0/validmind/ai.py

@@ -0,0 +1,193 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import concurrent.futures
+import os
+
+from openai import AzureOpenAI, OpenAI
+
+SYSTEM_PROMPT = """
+You are an expert data scientist and MRM specialist tasked with providing concise and'
+objective insights based on the results of quantitative model or dataset analysis.
+
+Examine the provided statistical test results and compose a brief summary. Highlight crucial
+insights, focusing on the distribution characteristics, central tendencies (such as mean or median),
+and the variability (including standard deviation and range) of the metrics. Evaluate how
+these statistics might influence the development and performance of a predictive model. Identify
+and explain any discernible trends or anomalies in the test results.
+
+Your analysis will act as the description of the result in the model documentation.
+
+Avoid long sentences and complex vocabulary.
+Structure the response clearly and logically.
+Use valid Markdown syntax to format the response (tables are supported).
+Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
+Use the following format for the response (feel free to modify slightly if necessary):
+```
+**<Test Name>** <continue to explain what it does in detail>...
+
+The results of this test <detailed explanation of the results>...
+
+In summary the following key insights can be gained:
+
+- **<key insight 1 - title>**: <explanation of key insight 1>
+- ...<continue with any other key insights using the same format>
+```
+It is very important that the text is nicely formatted and contains enough information to be useful to the user as documentation.
+""".strip()
+USER_PROMPT = """
+Test ID: {test_name}
+Test Description: {test_description}
+Test Results (the raw results of the test):
+{test_results}
+Test Summary (what the user sees in the documentation):
+{test_summary}
+""".strip()
+USER_PROMPT_FIGURES = """
+Test ID: {test_name}
+Test Description: {test_description}
+The attached plots show the results of the test.
+""".strip()
+
+__client = None
+__model = None
+
+__executor = concurrent.futures.ThreadPoolExecutor()
+
+
+def __get_client_and_model():
+    """
+    Get the model to use for generating interpretations
+    """
+    global __client, __model
+
+    if __client and __model:
+        return __client, __model
+
+    if "OPENAI_API_KEY" in os.environ:
+        __client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+        __model = os.environ.get("VM_OPENAI_MODEL", "gpt-4-turbo")
+
+    elif "AZURE_OPENAI_KEY" in os.environ:
+        if "AZURE_OPENAI_ENDPOINT" not in os.environ:
+            raise ValueError(
+                "AZURE_OPENAI_ENDPOINT must be set to run LLM tests with Azure"
+            )
+
+        if "AZURE_OPENAI_MODEL" not in os.environ:
+            raise ValueError(
+                "AZURE_OPENAI_MODEL must be set to run LLM tests with Azure"
+            )
+
+        __client = AzureOpenAI(
+            azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"),
+            api_key=os.environ.get("AZURE_OPENAI_KEY"),
+            api_version=os.environ.get("AZURE_OPENAI_VERSION", "2023-05-15"),
+        )
+        __model = os.environ.get("AZURE_OPENAI_MODEL")
+
+    else:
+        raise ValueError("OPENAI_API_KEY or AZURE_OPENAI_KEY must be set")
+
+    return __client, __model
+
+
+class DescriptionFuture:
+    """This will be immediately returned from generate_description so that
+    the tests can continue to be run in parallel while the description is
+    retrieved asynchronously.
+
+    The value will be retrieved later and if its not ready yet, it should
+    block until it is.
+    """
+
+    def __init__(self, future):
+        self._future = future
+
+    def get_description(self):
+        # This will block until the future is completed
+        return self._future.result()
+
+
+def generate_description_async(
+    test_name: str,
+    test_description: str,
+    test_results: str,
+    test_summary: str,
+    figures: list = None,
+):
+    """Generate the description for the test results"""
+    client, _ = __get_client_and_model()
+
+    # get last part of test id
+    test_name = test_name.split(".")[-1]
+
+    if not test_results and not test_summary:
+        if not figures:
+            raise ValueError("No results, summary or figures provided")
+
+        response = client.chat.completions.create(
+            model="gpt-4-turbo",
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": USER_PROMPT_FIGURES.format(
+                                test_name=test_name,
+                                test_description=test_description,
+                            ),
+                        },
+                        *[
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": figure._get_b64_url(),
+                                },
+                            }
+                            for figure in figures
+                        ],
+                    ],
+                },
+            ],
+        )
+    else:
+        response = client.chat.completions.create(
+            model="gpt-4-turbo",
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {
+                    "role": "user",
+                    "content": USER_PROMPT.format(
+                        test_name=test_name,
+                        test_description=test_description,
+                        test_results=test_results,
+                        test_summary=test_summary,
+                    ),
+                },
+            ],
+        )
+
+    return response.choices[0].message.content.strip("```").strip()
+
+
+def generate_description(
+    test_name: str,
+    test_description: str,
+    test_results: str,
+    test_summary: str,
+    figures: list = None,
+):
+    future = __executor.submit(
+        generate_description_async,
+        test_name,
+        test_description,
+        test_results,
+        test_summary,
+        figures,
+    )
+
+    return DescriptionFuture(future)

{validmind-2.0.1 → validmind-2.1.0}/validmind/api_client.py

@@ -16,6 +16,7 @@ from io import BytesIO
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import aiohttp
+import mistune
 import requests
 from aiohttp import FormData
 
@@ -171,6 +172,18 @@ def __ping() -> Dict[str, Any]:
     )
 
 
+def reload():
+    """Reconnect to the ValidMind API and reload the project configuration"""
+
+    try:
+        __ping()
+    except Exception as e:
+        # if the api host is https, assume we're not in dev mode and send to sentry
+        if _api_host.startswith("https://"):
+            send_single_error(e)
+        raise e
+
+
 async def __get_url(endpoint: str, params: Optional[Dict[str, str]] = None) -> str:
     if not _run_cuid:
         start_run()
@@ -282,45 +295,46 @@ async def log_figures(figures: List[Figure]) -> Dict[str, Any]:
     Returns:
         dict: The response from the API
     """
-    if client_config.can_log_figures():  # check if the backend supports batch logging
-        try:
-            data = {}
-            files = {}
-            for figure in figures:
-                data.update(
-                    {f"{k}-{figure.key}": v for k, v in figure.serialize().items()}
-                )
-                files.update(
-                    {
-                        f"{k}-{figure.key}": v
-                        for k, v in figure.serialize_files().items()
-                    }
-                )
-
-            return await _post(
-                "log_figures",
-                data=data,
-                files=files,
-            )
-        except Exception as e:
-            logger.error("Error logging figures to ValidMind API")
-            raise e
-
-    else:
-        return await asyncio.gather(*[log_figure(figure) for figure in figures])
+    # this actually slows things down - better to log them in parallel
+    # if client_config.can_log_figures():  # check if the backend supports batch logging
+    #     try:
+    #         data = {}
+    #         files = {}
+    #         for figure in figures:
+    #             data.update(
+    #                 {f"{k}-{figure.key}": v for k, v in figure.serialize().items()}
+    #             )
+    #             files.update(
+    #                 {
+    #                     f"{k}-{figure.key}": v
+    #                     for k, v in figure.serialize_files().items()
+    #                 }
+    #             )
+
+    #         return await _post(
+    #             "log_figures",
+    #             data=data,
+    #             files=files,
+    #         )
+    #     except Exception as e:
+    #         logger.error("Error logging figures to ValidMind API")
+    #         raise e
+
+    # else:
+    return await asyncio.gather(*[log_figure(figure) for figure in figures])
 
 
 async def log_metadata(
     content_id: str,
     text: Optional[str] = None,
-    extra_json: Optional[Dict[str, Any]] = None,
+    _json: Optional[Dict[str, Any]] = None,
 ) -> Dict[str, Any]:
     """Logs free-form metadata to ValidMind API.
 
     Args:
         content_id (str): Unique content identifier for the metadata
         text (str, optional): Free-form text to assign to the metadata. Defaults to None.
-        extra_json (dict, optional): Free-form key-value pairs to assign to the metadata. Defaults to None.
+        _json (dict, optional): Free-form key-value pairs to assign to the metadata. Defaults to None.
 
     Raises:
         Exception: If the API call fails
@@ -330,9 +344,9 @@ async def log_metadata(
     """
     metadata_dict = {"content_id": content_id}
     if text is not None:
-        metadata_dict["text"] = text
-    if extra_json is not None:
-        metadata_dict["extra_json"] = extra_json
+        metadata_dict["text"] = mistune.html(text)
+    if _json is not None:
+        metadata_dict["json"] = _json
 
     try:
         return await _post(

{validmind-2.0.1 → validmind-2.1.0}/validmind/client.py

@@ -7,6 +7,7 @@ Client interface for all data and model validation functions
 """
 
 import pandas as pd
+import polars as pl
 
 from .api_client import _log_input as log_input
 from .client_config import client_config
@@ -26,7 +27,13 @@ from .template import preview_template as _preview_template
 from .test_suites import get_by_id as get_test_suite_by_id
 from .utils import get_dataset_info, get_model_info
 from .vm_models import TestInput, TestSuite, TestSuiteRunner
-from .vm_models.dataset import DataFrameDataset, NumpyDataset, TorchDataset, VMDataset
+from .vm_models.dataset import (
+    DataFrameDataset,
+    NumpyDataset,
+    PolarsDataset,
+    TorchDataset,
+    VMDataset,
+)
 from .vm_models.model import VMModel, get_model_class
 
 pd.option_context("format.precision", 2)
@@ -54,11 +61,16 @@ def init_dataset(
     """
     Initializes a VM Dataset, which can then be passed to other functions
     that can perform additional analysis and tests on the data. This function
-    also ensures we are reading a valid dataset type. We only support Pandas
-    DataFrames at the moment.
+    also ensures we are reading a valid dataset type.
+
+    The following dataset types are supported:
+    - Pandas DataFrame
+    - Polars DataFrame
+    - Numpy ndarray
+    - Torch TensorDataset
 
     Args:
-        dataset (pd.DataFrame): We only support Pandas DataFrames at the moment
+        dataset : dataset from various python libraries
         model (VMModel): ValidMind model object
         options (dict): A dictionary of options for the dataset
         targets (vm.vm.DatasetTargets): A list of target variables
@@ -89,7 +101,7 @@ def init_dataset(
     input_id = input_id or "dataset"
 
     # Instantiate supported dataset types here
-    if dataset_class == "DataFrame":
+    if isinstance(dataset, pd.DataFrame):
         logger.info("Pandas dataset detected. Initializing VM Dataset instance...")
         vm_dataset = DataFrameDataset(
             input_id=input_id,
@@ -102,6 +114,19 @@ def init_dataset(
             target_class_labels=class_labels,
             date_time_index=date_time_index,
         )
+    elif isinstance(dataset, pl.DataFrame):
+        logger.info("Polars dataset detected. Initializing VM Dataset instance...")
+        vm_dataset = PolarsDataset(
+            input_id=input_id,
+            raw_dataset=dataset,
+            model=model,
+            target_column=target_column,
+            feature_columns=feature_columns,
+            text_column=text_column,
+            extra_columns=extra_columns,
+            target_class_labels=class_labels,
+            date_time_index=date_time_index,
+        )
     elif dataset_class == "ndarray":
         logger.info("Numpy ndarray detected. Initializing VM Dataset instance...")
         vm_dataset = NumpyDataset(
@@ -360,7 +385,7 @@ def preview_template():
 
 
 def run_documentation_tests(
-    section=None, send=True, fail_fast=False, inputs=None, **kwargs
+    section=None, send=True, fail_fast=False, inputs=None, config=None, **kwargs
 ):
     """Collect and run all the tests associated with a template
 
@@ -373,6 +398,7 @@ def run_documentation_tests(
         send (bool, optional): Whether to send the results to the ValidMind API. Defaults to True.
         fail_fast (bool, optional): Whether to stop running tests after the first failure. Defaults to False.
         inputs (dict, optional): A dictionary of test inputs to pass to the TestSuite
+        config: A dictionary of test parameters to override the defaults
         **kwargs: backwards compatibility for passing in test inputs using keyword arguments
 
     Returns:
@@ -401,6 +427,7 @@ def run_documentation_tests(
             send=send,
             fail_fast=fail_fast,
             inputs=inputs,
+            config=config,
             **kwargs,
         )
         test_suites[_section] = test_suite

{validmind-2.0.1 → validmind-2.1.0}/validmind/datasets/classification/customer_churn.py

@@ -60,7 +60,7 @@ def preprocess(df):
     return train_df, validation_df, test_df
 
 
-def get_demo_test_config():
+def get_demo_test_config(test_suite=None):
     """
     Returns input configuration for the default documentation
     template assigned to this demo model
@@ -81,7 +81,7 @@ def get_demo_test_config():
     - The only exception is ClassifierPerformance since that runs twice: once
         with the train_dataset (in sample) and once with the test_dataset (out of sample)
     """
-    default_config = vm.get_test_suite().get_default_config()
+    default_config = (test_suite or vm.get_test_suite()).get_default_config()
 
     for _, test_config in default_config.items():
         if "model" in test_config["inputs"]:

validmind-2.1.0/validmind/datasets/credit_risk/__init__.py

@@ -0,0 +1,11 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""
+Entrypoint for credit risk datasets.
+"""
+
+__all__ = [
+    "lending_club",
+]