validmind 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

validmind/__init__.py

@@ -50,7 +50,7 @@ from .__version__ import __version__  # noqa: E402
 from .api_client import init
 from .api_client import log_figure as _log_figure_async
 from .api_client import log_metrics as _log_metrics_async
-from .api_client import log_test_results
+from .api_client import log_test_results, reload
 from .client import (  # noqa: E402
     get_test_suite,
     init_dataset,
@@ -60,7 +60,7 @@ from .client import (  # noqa: E402
     run_documentation_tests,
     run_test_suite,
 )
-from .unit_metrics import run_metric
+from .tests.decorator import metric, tags, tasks
 from .utils import run_async  # noqa: E402
 
 
@@ -105,9 +105,13 @@ __all__ = [  # noqa
     "init_dataset",
     "init_model",
     "init_r_model",
+    "metric",
     "preview_template",
+    "reload",
     "run_documentation_tests",
     "run_test_suite",
+    "tags",
+    "tasks",
     "tests",
     "test_suites",
     "vm_models",
@@ -116,5 +120,4 @@ __all__ = [  # noqa
     "log_figure",
     "log_metrics",
     "log_test_results",
-    "run_metric",
 ]

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.0.1"
+__version__ = "2.1.0"

validmind/ai.py

@@ -0,0 +1,193 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import concurrent.futures
+import os
+
+from openai import AzureOpenAI, OpenAI
+
+SYSTEM_PROMPT = """
+You are an expert data scientist and MRM specialist tasked with providing concise and'
+objective insights based on the results of quantitative model or dataset analysis.
+
+Examine the provided statistical test results and compose a brief summary. Highlight crucial
+insights, focusing on the distribution characteristics, central tendencies (such as mean or median),
+and the variability (including standard deviation and range) of the metrics. Evaluate how
+these statistics might influence the development and performance of a predictive model. Identify
+and explain any discernible trends or anomalies in the test results.
+
+Your analysis will act as the description of the result in the model documentation.
+
+Avoid long sentences and complex vocabulary.
+Structure the response clearly and logically.
+Use valid Markdown syntax to format the response (tables are supported).
+Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
+Use the following format for the response (feel free to modify slightly if necessary):
+```
+**<Test Name>** <continue to explain what it does in detail>...
+
+The results of this test <detailed explanation of the results>...
+
+In summary the following key insights can be gained:
+
+- **<key insight 1 - title>**: <explanation of key insight 1>
+- ...<continue with any other key insights using the same format>
+```
+It is very important that the text is nicely formatted and contains enough information to be useful to the user as documentation.
+""".strip()
+USER_PROMPT = """
+Test ID: {test_name}
+Test Description: {test_description}
+Test Results (the raw results of the test):
+{test_results}
+Test Summary (what the user sees in the documentation):
+{test_summary}
+""".strip()
+USER_PROMPT_FIGURES = """
+Test ID: {test_name}
+Test Description: {test_description}
+The attached plots show the results of the test.
+""".strip()
+
+__client = None
+__model = None
+
+__executor = concurrent.futures.ThreadPoolExecutor()
+
+
+def __get_client_and_model():
+    """
+    Get the model to use for generating interpretations
+    """
+    global __client, __model
+
+    if __client and __model:
+        return __client, __model
+
+    if "OPENAI_API_KEY" in os.environ:
+        __client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+        __model = os.environ.get("VM_OPENAI_MODEL", "gpt-4-turbo")
+
+    elif "AZURE_OPENAI_KEY" in os.environ:
+        if "AZURE_OPENAI_ENDPOINT" not in os.environ:
+            raise ValueError(
+                "AZURE_OPENAI_ENDPOINT must be set to run LLM tests with Azure"
+            )
+
+        if "AZURE_OPENAI_MODEL" not in os.environ:
+            raise ValueError(
+                "AZURE_OPENAI_MODEL must be set to run LLM tests with Azure"
+            )
+
+        __client = AzureOpenAI(
+            azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"),
+            api_key=os.environ.get("AZURE_OPENAI_KEY"),
+            api_version=os.environ.get("AZURE_OPENAI_VERSION", "2023-05-15"),
+        )
+        __model = os.environ.get("AZURE_OPENAI_MODEL")
+
+    else:
+        raise ValueError("OPENAI_API_KEY or AZURE_OPENAI_KEY must be set")
+
+    return __client, __model
+
+
+class DescriptionFuture:
+    """This will be immediately returned from generate_description so that
+    the tests can continue to be run in parallel while the description is
+    retrieved asynchronously.
+
+    The value will be retrieved later and if its not ready yet, it should
+    block until it is.
+    """
+
+    def __init__(self, future):
+        self._future = future
+
+    def get_description(self):
+        # This will block until the future is completed
+        return self._future.result()
+
+
+def generate_description_async(
+    test_name: str,
+    test_description: str,
+    test_results: str,
+    test_summary: str,
+    figures: list = None,
+):
+    """Generate the description for the test results"""
+    client, _ = __get_client_and_model()
+
+    # get last part of test id
+    test_name = test_name.split(".")[-1]
+
+    if not test_results and not test_summary:
+        if not figures:
+            raise ValueError("No results, summary or figures provided")
+
+        response = client.chat.completions.create(
+            model="gpt-4-turbo",
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": USER_PROMPT_FIGURES.format(
+                                test_name=test_name,
+                                test_description=test_description,
+                            ),
+                        },
+                        *[
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": figure._get_b64_url(),
+                                },
+                            }
+                            for figure in figures
+                        ],
+                    ],
+                },
+            ],
+        )
+    else:
+        response = client.chat.completions.create(
+            model="gpt-4-turbo",
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {
+                    "role": "user",
+                    "content": USER_PROMPT.format(
+                        test_name=test_name,
+                        test_description=test_description,
+                        test_results=test_results,
+                        test_summary=test_summary,
+                    ),
+                },
+            ],
+        )
+
+    return response.choices[0].message.content.strip("```").strip()
+
+
+def generate_description(
+    test_name: str,
+    test_description: str,
+    test_results: str,
+    test_summary: str,
+    figures: list = None,
+):
+    future = __executor.submit(
+        generate_description_async,
+        test_name,
+        test_description,
+        test_results,
+        test_summary,
+        figures,
+    )
+
+    return DescriptionFuture(future)

validmind/api_client.py

@@ -16,6 +16,7 @@ from io import BytesIO
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import aiohttp
+import mistune
 import requests
 from aiohttp import FormData
 
@@ -171,6 +172,18 @@ def __ping() -> Dict[str, Any]:
     )
 
 
+def reload():
+    """Reconnect to the ValidMind API and reload the project configuration"""
+
+    try:
+        __ping()
+    except Exception as e:
+        # if the api host is https, assume we're not in dev mode and send to sentry
+        if _api_host.startswith("https://"):
+            send_single_error(e)
+        raise e
+
+
 async def __get_url(endpoint: str, params: Optional[Dict[str, str]] = None) -> str:
     if not _run_cuid:
         start_run()
@@ -282,45 +295,46 @@ async def log_figures(figures: List[Figure]) -> Dict[str, Any]:
     Returns:
         dict: The response from the API
     """
-    if client_config.can_log_figures():  # check if the backend supports batch logging
-        try:
-            data = {}
-            files = {}
-            for figure in figures:
-                data.update(
-                    {f"{k}-{figure.key}": v for k, v in figure.serialize().items()}
-                )
-                files.update(
-                    {
-                        f"{k}-{figure.key}": v
-                        for k, v in figure.serialize_files().items()
-                    }
-                )
-
-            return await _post(
-                "log_figures",
-                data=data,
-                files=files,
-            )
-        except Exception as e:
-            logger.error("Error logging figures to ValidMind API")
-            raise e
-
-    else:
-        return await asyncio.gather(*[log_figure(figure) for figure in figures])
+    # this actually slows things down - better to log them in parallel
+    # if client_config.can_log_figures():  # check if the backend supports batch logging
+    #     try:
+    #         data = {}
+    #         files = {}
+    #         for figure in figures:
+    #             data.update(
+    #                 {f"{k}-{figure.key}": v for k, v in figure.serialize().items()}
+    #             )
+    #             files.update(
+    #                 {
+    #                     f"{k}-{figure.key}": v
+    #                     for k, v in figure.serialize_files().items()
+    #                 }
+    #             )
+
+    #         return await _post(
+    #             "log_figures",
+    #             data=data,
+    #             files=files,
+    #         )
+    #     except Exception as e:
+    #         logger.error("Error logging figures to ValidMind API")
+    #         raise e
+
+    # else:
+    return await asyncio.gather(*[log_figure(figure) for figure in figures])
 
 
 async def log_metadata(
     content_id: str,
     text: Optional[str] = None,
-    extra_json: Optional[Dict[str, Any]] = None,
+    _json: Optional[Dict[str, Any]] = None,
 ) -> Dict[str, Any]:
     """Logs free-form metadata to ValidMind API.
 
     Args:
         content_id (str): Unique content identifier for the metadata
         text (str, optional): Free-form text to assign to the metadata. Defaults to None.
-        extra_json (dict, optional): Free-form key-value pairs to assign to the metadata. Defaults to None.
+        _json (dict, optional): Free-form key-value pairs to assign to the metadata. Defaults to None.
 
     Raises:
         Exception: If the API call fails
@@ -330,9 +344,9 @@ async def log_metadata(
     """
     metadata_dict = {"content_id": content_id}
     if text is not None:
-        metadata_dict["text"] = text
-    if extra_json is not None:
-        metadata_dict["extra_json"] = extra_json
+        metadata_dict["text"] = mistune.html(text)
+    if _json is not None:
+        metadata_dict["json"] = _json
 
     try:
         return await _post(

validmind/client.py

@@ -7,6 +7,7 @@ Client interface for all data and model validation functions
 """
 
 import pandas as pd
+import polars as pl
 
 from .api_client import _log_input as log_input
 from .client_config import client_config
@@ -26,7 +27,13 @@ from .template import preview_template as _preview_template
 from .test_suites import get_by_id as get_test_suite_by_id
 from .utils import get_dataset_info, get_model_info
 from .vm_models import TestInput, TestSuite, TestSuiteRunner
-from .vm_models.dataset import DataFrameDataset, NumpyDataset, TorchDataset, VMDataset
+from .vm_models.dataset import (
+    DataFrameDataset,
+    NumpyDataset,
+    PolarsDataset,
+    TorchDataset,
+    VMDataset,
+)
 from .vm_models.model import VMModel, get_model_class
 
 pd.option_context("format.precision", 2)
@@ -54,11 +61,16 @@ def init_dataset(
     """
     Initializes a VM Dataset, which can then be passed to other functions
     that can perform additional analysis and tests on the data. This function
-    also ensures we are reading a valid dataset type. We only support Pandas
-    DataFrames at the moment.
+    also ensures we are reading a valid dataset type.
+
+    The following dataset types are supported:
+    - Pandas DataFrame
+    - Polars DataFrame
+    - Numpy ndarray
+    - Torch TensorDataset
 
     Args:
-        dataset (pd.DataFrame): We only support Pandas DataFrames at the moment
+        dataset : dataset from various python libraries
         model (VMModel): ValidMind model object
         options (dict): A dictionary of options for the dataset
         targets (vm.vm.DatasetTargets): A list of target variables
@@ -89,7 +101,7 @@ def init_dataset(
     input_id = input_id or "dataset"
 
     # Instantiate supported dataset types here
-    if dataset_class == "DataFrame":
+    if isinstance(dataset, pd.DataFrame):
         logger.info("Pandas dataset detected. Initializing VM Dataset instance...")
         vm_dataset = DataFrameDataset(
             input_id=input_id,
@@ -102,6 +114,19 @@ def init_dataset(
             target_class_labels=class_labels,
             date_time_index=date_time_index,
         )
+    elif isinstance(dataset, pl.DataFrame):
+        logger.info("Polars dataset detected. Initializing VM Dataset instance...")
+        vm_dataset = PolarsDataset(
+            input_id=input_id,
+            raw_dataset=dataset,
+            model=model,
+            target_column=target_column,
+            feature_columns=feature_columns,
+            text_column=text_column,
+            extra_columns=extra_columns,
+            target_class_labels=class_labels,
+            date_time_index=date_time_index,
+        )
     elif dataset_class == "ndarray":
         logger.info("Numpy ndarray detected. Initializing VM Dataset instance...")
         vm_dataset = NumpyDataset(
@@ -360,7 +385,7 @@ def preview_template():
 
 
 def run_documentation_tests(
-    section=None, send=True, fail_fast=False, inputs=None, **kwargs
+    section=None, send=True, fail_fast=False, inputs=None, config=None, **kwargs
 ):
     """Collect and run all the tests associated with a template
 
@@ -373,6 +398,7 @@ def run_documentation_tests(
         send (bool, optional): Whether to send the results to the ValidMind API. Defaults to True.
         fail_fast (bool, optional): Whether to stop running tests after the first failure. Defaults to False.
         inputs (dict, optional): A dictionary of test inputs to pass to the TestSuite
+        config: A dictionary of test parameters to override the defaults
         **kwargs: backwards compatibility for passing in test inputs using keyword arguments
 
     Returns:
@@ -401,6 +427,7 @@ def run_documentation_tests(
             send=send,
             fail_fast=fail_fast,
             inputs=inputs,
+            config=config,
             **kwargs,
         )
         test_suites[_section] = test_suite

validmind/datasets/classification/customer_churn.py

@@ -60,7 +60,7 @@ def preprocess(df):
     return train_df, validation_df, test_df
 
 
-def get_demo_test_config():
+def get_demo_test_config(test_suite=None):
     """
     Returns input configuration for the default documentation
     template assigned to this demo model
@@ -81,7 +81,7 @@ def get_demo_test_config():
     - The only exception is ClassifierPerformance since that runs twice: once
         with the train_dataset (in sample) and once with the test_dataset (out of sample)
     """
-    default_config = vm.get_test_suite().get_default_config()
+    default_config = (test_suite or vm.get_test_suite()).get_default_config()
 
     for _, test_config in default_config.items():
         if "model" in test_config["inputs"]:

validmind/datasets/credit_risk/__init__.py

@@ -0,0 +1,11 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""
+Entrypoint for credit risk datasets.
+"""
+
+__all__ = [
+    "lending_club",
+]