validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/__init__.py

@@ -3,25 +3,19 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-ValidMind’s Python Library is a library of developer tools and methods designed to automate
-the documentation and validation of your models.
+The ValidMind Library is a suite of developer tools and methods designed to automate the documentation and validation of your models.
 
-The Library is designed to be model agnostic. If your model is built in Python, ValidMind's
-Python library will provide all the standard functionality without requiring your developers to rewrite any functions.
+Designed to be model agnostic, the ValidMind Library provides all the standard functionality without requiring you to rewrite any functions as long as your model is built in Python.
 
-The Library provides a rich suite of documentation tools and test suites, from documenting
-descriptions of your dataset to testing your models for weak spots and overfit areas. The Library
-helps you automate the generation of model documentation by feeding the ValidMind platform with documentation
-artifacts and test results to the ValidMind platform.
+With a rich array of documentation tools and test suites, from documenting descriptions of your datasets to testing your models for weak spots and overfit areas, the ValidMind Library helps you automate model documentation by feeding the ValidMind Platform with documentation artifacts and test results.
 
-To install the client library:
+To install the ValidMind Library:
 
 ```bash
 pip install validmind
 ```
 
-To initialize the client library, paste the code snippet with the client integration details directly into your
-development source code, replacing this example with your own:
+To initialize the ValidMind Library, paste the code snippet with the model identifier credentials directly into your development source code, replacing this example with your own:
 
 ```python
 import validmind as vm
@@ -34,9 +28,7 @@ vm.init(
 )
 ```
 
-After you have pasted the code snippet into your development source code and executed the code, the Python client
-library will register with ValidMind. You can now use the Library to document and test your models,
-and to upload to the ValidMind Platform.
+After you have pasted the code snippet into your development source code and executed the code, the Python Library API will register with ValidMind. You can now use the ValidMind Library to document and test your models, and to upload to the ValidMind Platform.
 """
 import warnings
 
@@ -57,11 +49,11 @@ from .client import (  # noqa: E402
     run_documentation_tests,
     run_test_suite,
 )
-from .tests.decorator import metric, tags, tasks, test
+from .tests.decorator import tags, tasks, test
 
 __all__ = [  # noqa
     "__version__",
-    # Framework High Level API
+    # Python Library API
     "datasets",
     "errors",
     "get_test_suite",
@@ -69,7 +61,6 @@ __all__ = [  # noqa
     "init_dataset",
     "init_model",
     "init_r_model",
-    "metric",
     "preview_template",
     "reload",
     "run_documentation_tests",

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.5.24"
+__version__ = "2.6.7"

validmind/ai/test_descriptions.py

@@ -2,17 +2,20 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import json
 import os
 import re
 from concurrent.futures import ThreadPoolExecutor
-from typing import Union
+from typing import List, Optional, Union
 
 from jinja2 import Template
 
-from validmind.utils import md_to_html
-
 from ..client_config import client_config
 from ..logging import get_logger
+from ..utils import NumpyEncoder, md_to_html, test_id_to_name
+from ..vm_models.figure import Figure
+from ..vm_models.result import ResultTable
+from .utils import DescriptionFuture, get_client_and_model
 
 __executor = ThreadPoolExecutor()
 __prompt = None
@@ -20,10 +23,6 @@ __prompt = None
 logger = get_logger(__name__)
 
 
-AI_REVISION_NAME = "Generated by ValidMind AI"
-DEFAULT_REVISION_NAME = "Default Description"
-
-
 def _load_prompt():
     global __prompt
 
@@ -55,7 +54,6 @@ def prompt_to_message(role, prompt):
         if start > last_index:
             content.append({"type": "text", "text": prompt[last_index:start]})
 
-        # Image
         content.append({"type": "image_url", "image_url": {"url": match.group(1)}})
 
         last_index = end
@@ -67,79 +65,66 @@ def prompt_to_message(role, prompt):
     return {"role": role, "content": content}
 
 
-class DescriptionFuture:
-    """This will be immediately returned from generate_description so that
-    the tests can continue to be run in parallel while the description is
-    retrieved asynchronously.
-
-    The value will be retrieved later and if its not ready yet, it should
-    block until it is.
-    """
-
-    def __init__(self, future):
-        self._future = future
-
-    def get_description(self):
-        if isinstance(self._future, str):
-            description = self._future
-        else:
-            # This will block until the future is completed
-            description = self._future.result()
-
-        return md_to_html(description, mathml=True)
-
-
 def generate_description(
     test_id: str,
     test_description: str,
-    test_summary: str,
+    tables: List[ResultTable] = None,
     metric: Union[float, int] = None,
-    figures: list = None,
+    figures: List[Figure] = None,
+    title: Optional[str] = None,
 ):
     """Generate the description for the test results"""
-    if not test_summary and not figures and not metric:
+    if not tables and not figures and not metric:
         raise ValueError(
-            "No summary, unit metric or figures provided - cannot generate description"
+            "No tables, unit metric or figures provided - cannot generate description"
         )
 
-    # TODO: fix circular import
-    from validmind.ai.utils import get_client_and_model
+    # # TODO: fix circular import
+    # from validmind.ai.utils import get_client_and_model
 
     client, model = get_client_and_model()
 
     # get last part of test id
-    test_name = test_id.split(".")[-1]
-    # truncate the test description to save time
-    test_description = (
-        f"{test_description[:500]}..."
-        if len(test_description) > 500
-        else test_description
-    )
-
-    if metric:
-        metric_summary = f"**Metric Value**: {metric}"
-        if test_summary:
-            test_summary = metric_summary + "\n" + test_summary
-        else:
-            test_summary = metric_summary
+    test_name = title or test_id.split(".")[-1]
+
+    # TODO: fully support metrics
+    if metric is not None:
+        tables = [] if not tables else tables
+        tables.append(
+            ResultTable(
+                data=[
+                    {"Metric": test_id_to_name(test_id), "Value": metric},
+                ],
+            )
+        )
 
-    figures = [] if test_summary else figures
+    if tables:
+        summary = "\n---\n".join(
+            [
+                json.dumps(table.serialize(), cls=NumpyEncoder, separators=(",", ":"))
+                for table in tables
+            ]
+        )
+    else:
+        summary = None
 
     input_data = {
         "test_name": test_name,
         "test_description": test_description,
-        "summary": test_summary,
-        "figures": [figure._get_b64_url() for figure in figures],
+        "title": title,
+        "summary": summary,
+        "figures": [figure._get_b64_url() for figure in ([] if tables else figures)],
     }
     system, user = _load_prompt()
 
+    messages = [
+        prompt_to_message("system", system.render(input_data)),
+        prompt_to_message("user", user.render(input_data)),
+    ]
     response = client.chat.completions.create(
         model=model,
         temperature=0.0,
-        messages=[
-            prompt_to_message("system", system.render(input_data)),
-            prompt_to_message("user", user.render(input_data)),
-        ],
+        messages=messages,
     )
 
     return response.choices[0].message.content
@@ -148,18 +133,20 @@ def generate_description(
 def background_generate_description(
     test_id: str,
     test_description: str,
-    test_summary: str,
-    figures: list = None,
+    tables: List[ResultTable] = None,
+    figures: List[Figure] = None,
     metric: Union[int, float] = None,
+    title: Optional[str] = None,
 ):
     def wrapped():
         try:
             return generate_description(
                 test_id=test_id,
                 test_description=test_description,
-                test_summary=test_summary,
+                tables=tables,
                 figures=figures,
                 metric=metric,
+                title=title,
             )
         except Exception as e:
             logger.error(f"Failed to generate description: {e}")
@@ -169,14 +156,14 @@ def background_generate_description(
     return DescriptionFuture(__executor.submit(wrapped))
 
 
-def get_description_metadata(
-    test_id,
-    default_description,
-    summary=None,
-    figures=None,
-    metric=None,
-    prefix="metric_description",
-    should_generate=True,
+def get_result_description(
+    test_id: str,
+    test_description: str,
+    tables: List[ResultTable] = None,
+    figures: List[Figure] = None,
+    metric: Union[int, float] = None,
+    should_generate: bool = True,
+    title: Optional[str] = None,
 ):
     """Get Metadata Dictionary for a Test or Metric Result
 
@@ -184,24 +171,23 @@ def get_description_metadata(
     description and returns a metadata object that can be logged with the test results.
 
     By default, the description is generated by an LLM that will interpret the test
-    results and provide a human-readable description. If the summary or figures are
+    results and provide a human-readable description. If the tables or figures are
     not provided, or the `VALIDMIND_LLM_DESCRIPTIONS_ENABLED` environment variable is
     set to `0` or `false` or no LLM has been configured, the default description will
     be used as the test result description.
 
-    Note: Either the summary or figures must be provided to generate the description.
+    Note: Either the tables or figures must be provided to generate the description.
 
     Args:
         test_id (str): The test ID
-        default_description (str): The default description for the test
-        summary (Any): The test summary or results to interpret
+        test_description (str): The default description for the test
+        tables (Any): The test tables or results to interpret
         figures (List[Figure]): The figures to attach to the test suite result
         metric (Union[int, float]): Unit metrics attached to the test result
-        prefix (str): The prefix to use for the content ID (Default: "metric_description")
         should_generate (bool): Whether to generate the description or not (Default: True)
 
     Returns:
-        dict: The metadata object to be logged with the test results
+        str: The description to be logged with the test results
     """
     # Check the feature flag first, then the environment variable
     llm_descriptions_enabled = (
@@ -214,27 +200,22 @@ def get_description_metadata(
 
     if (
         should_generate
-        and (summary or figures)
+        and (tables or figures)
         and llm_descriptions_enabled
         and is_configured()
     ):
-        revision_name = AI_REVISION_NAME
-
         # get description future and set it as the description in the metadata
         # this will lazily retrieved so it can run in the background in parallel
         description = background_generate_description(
             test_id=test_id,
-            test_description=default_description,
-            test_summary=summary,
+            test_description=test_description,
+            tables=tables,
             figures=figures,
             metric=metric,
+            title=title,
         )
 
     else:
-        revision_name = DEFAULT_REVISION_NAME
-        description = md_to_html(default_description, mathml=True)
+        description = md_to_html(test_description, mathml=True)
 
-    return {
-        "content_id": f"{prefix}:{test_id}::{revision_name}",
-        "text": description,
-    }
+    return description

validmind/ai/test_result_description/context.py

@@ -37,11 +37,11 @@ class Context:
         pass
 
     def load(self, input_data):
-        # this task can accept a dict or a test result object from the library
+        # this task can accept a dict or a test result object from the ValidMind Library
         if isinstance(input_data, dict):
             return input_data
 
-        # we are likely running outside of the library and need to convert
+        # we are likely running outside of the ValidMind Library and need to convert
         # the test result object to a dictionary
         test_result = input_data
 

validmind/ai/utils.py

@@ -7,8 +7,8 @@ from urllib.parse import urljoin
 
 from openai import AzureOpenAI, Client, OpenAI
 
-from ..api_client import get_ai_key, get_api_host
 from ..logging import get_logger
+from ..utils import md_to_html
 
 logger = get_logger(__name__)
 
@@ -19,6 +19,28 @@ __model = None
 __ack = None
 
 
+class DescriptionFuture:
+    """This will be immediately returned from generate_description so that
+    the tests can continue to be run in parallel while the description is
+    retrieved asynchronously.
+
+    The value will be retrieved later and if its not ready yet, it should
+    block until it is.
+    """
+
+    def __init__(self, future):
+        self._future = future
+
+    def get_description(self):
+        if isinstance(self._future, str):
+            description = self._future
+        else:
+            # This will block until the future is completed
+            description = self._future.result()
+
+        return md_to_html(description, mathml=True)
+
+
 def get_client_and_model():
     """Get model and client to use for generating interpretations
 
@@ -58,6 +80,9 @@ def get_client_and_model():
 
     else:
         try:
+            # TODO: fix circular import
+            from ..api_client import get_ai_key, get_api_host
+
             response = get_ai_key()
             __client = Client(
                 base_url=(

validmind/api_client.py

@@ -23,9 +23,7 @@ from .client_config import client_config
 from .errors import MissingAPICredentialsError, MissingModelIdError, raise_api_error
 from .logging import get_logger, init_sentry, send_single_error
 from .utils import NumpyEncoder, run_async
-from .vm_models import Figure, MetricResult, ThresholdTestResults
-
-# TODO: can't import types from vm_models because of circular dependency
+from .vm_models import Figure
 
 logger = get_logger(__name__)
 
@@ -50,6 +48,14 @@ def _close_session():
                 loop.create_task(__api_session.close())
             else:
                 loop.run_until_complete(__api_session.close())
+        except RuntimeError as e:
+            # ignore RuntimeError when closing the session from the main thread
+            if "no current event loop in thread" in str(e):
+                pass
+            elif "Event loop is closed" in str(e):
+                pass
+            else:
+                raise e
         except Exception as e:
             logger.exception("Error closing aiohttp session at exit: %s", e)
 
@@ -187,7 +193,7 @@ def init(
     api_secret: Optional[str] = None,
     api_host: Optional[str] = None,
     model: Optional[str] = None,
-    monitoring=False,
+    monitoring: bool = False,
 ):
     """
     Initializes the API client instances and calls the /ping endpoint to ensure
@@ -202,7 +208,7 @@ def init(
         api_key (str, optional): The API key. Defaults to None.
         api_secret (str, optional): The API secret. Defaults to None.
         api_host (str, optional): The API host. Defaults to None.
-        monitoring (str, optional): The ongoing monitoring flag. Defaults to False.
+        monitoring (bool): The ongoing monitoring flag. Defaults to False.
 
     Raises:
         ValueError: If the API key and secret are not provided
@@ -212,7 +218,7 @@ def init(
     if api_key == "...":
         # special case to detect when running a notebook placeholder (...)
         # will override with environment variables for easier local development
-        api_host = api_key = api_secret = project = None
+        api_host = api_key = api_secret = project = model = None
 
     _model_cuid = project or model or os.getenv("VM_API_MODEL")
     if _model_cuid is None:
@@ -244,30 +250,7 @@ def reload():
         raise e
 
 
-async def log_figure(figure: Figure) -> Dict[str, Any]:
-    """Logs a figure
-
-    Args:
-        figure (Figure): The Figure object wrapper
-
-    Raises:
-        Exception: If the API call fails
-
-    Returns:
-        dict: The response from the API
-    """
-    try:
-        return await _post(
-            "log_figure",
-            data=figure.serialize(),
-            files=figure.serialize_files(),
-        )
-    except Exception as e:
-        logger.error("Error logging figure to ValidMind API")
-        raise e
-
-
-async def get_metadata(content_id: str) -> Dict[str, Any]:
+async def aget_metadata(content_id: str) -> Dict[str, Any]:
     """Gets a metadata object from ValidMind API.
 
     Args:
@@ -279,11 +262,10 @@ async def get_metadata(content_id: str) -> Dict[str, Any]:
     Returns:
         dict: Metadata object
     """
-    # TODO: add a more accurate type hint/documentation
     return await _get(f"get_metadata/{content_id}")
 
 
-async def log_metadata(
+async def alog_metadata(
     content_id: str,
     text: Optional[str] = None,
     _json: Optional[Dict[str, Any]] = None,
@@ -317,21 +299,11 @@ async def log_metadata(
         raise e
 
 
-async def log_metric_result(
-    metric: MetricResult,
-    inputs: List[str],
-    output_template: str = None,
-    section_id: str = None,
-    position: int = None,
-) -> Dict[str, Any]:
-    """Logs metrics to ValidMind API.
+async def alog_figure(figure: Figure) -> Dict[str, Any]:
+    """Logs a figure
 
     Args:
-        metric (MetricResult): A MetricResult object
-        inputs (list): A list of input keys (names) that were used to run the test
-        output_template (str): The optional output template for the test
-        section_id (str): The section ID add a test driven block to the documentation
-        position (int): The position in the section to add the test driven block
+        figure (Figure): The Figure object wrapper
 
     Raises:
         Exception: If the API call fails
@@ -339,33 +311,19 @@ async def log_metric_result(
     Returns:
         dict: The response from the API
     """
-    request_params = {}
-    if section_id:
-        request_params["section_id"] = section_id
-    if position is not None:
-        request_params["position"] = position
-
-    metric_data = {
-        **metric.serialize(),
-        "inputs": inputs,
-    }
-    if output_template:
-        metric_data["output_template"] = output_template
-
     try:
         return await _post(
-            "log_metrics",
-            params=request_params,
-            data=json.dumps([metric_data], cls=NumpyEncoder, allow_nan=False),
+            "log_figure",
+            data=figure.serialize(),
+            files=figure.serialize_files(),
         )
     except Exception as e:
-        logger.error("Error logging metrics to ValidMind API")
+        logger.error("Error logging figure to ValidMind API")
         raise e
 
 
-async def log_test_result(
-    result: ThresholdTestResults,
-    inputs: List[str],
+async def alog_test_result(
+    result: Dict[str, Any],
     section_id: str = None,
     position: int = None,
 ) -> Dict[str, Any]:
@@ -375,8 +333,7 @@ async def log_test_result(
     can also be called directly if the user wants to run tests on their own.
 
     Args:
-        result (validmind.ThresholdTestResults): A ThresholdTestResults object
-        inputs (list): A list of input keys (names) that were used to run the test
+        result (dict): A dictionary representing the test result
         section_id (str, optional): The section ID add a test driven block to the documentation
         position (int): The position in the section to add the test driven block
 
@@ -391,16 +348,12 @@ async def log_test_result(
         request_params["section_id"] = section_id
     if position is not None:
         request_params["position"] = position
-
     try:
         return await _post(
             "log_test_results",
             params=request_params,
             data=json.dumps(
-                {
-                    **result.serialize(),
-                    "inputs": inputs,
-                },
+                result,
                 cls=NumpyEncoder,
                 allow_nan=False,
             ),
@@ -410,7 +363,9 @@ async def log_test_result(
         raise e
 
 
-def log_input(input_id: str, type: str, metadata: Dict[str, Any]) -> Dict[str, Any]:
+async def alog_input(
+    input_id: str, type: str, metadata: Dict[str, Any]
+) -> Dict[str, Any]:
     """Logs input information - internal use for now (don't expose via public API)
 
     Args:
@@ -425,8 +380,7 @@ def log_input(input_id: str, type: str, metadata: Dict[str, Any]) -> Dict[str, A
         dict: The response from the API
     """
     try:
-        return run_async(
-            _post,
+        return await _post(
             "log_input",
             data=json.dumps(
                 {
@@ -443,9 +397,13 @@ def log_input(input_id: str, type: str, metadata: Dict[str, Any]) -> Dict[str, A
         raise e
 
 
+def log_input(input_id: str, type: str, metadata: Dict[str, Any]) -> Dict[str, Any]:
+    return run_async(alog_input, input_id, type, metadata)
+
+
 async def alog_metric(
     key: str,
-    value: float,
+    value: Union[int, float],
     inputs: Optional[List[str]] = None,
     params: Optional[Dict[str, Any]] = None,
     recorded_at: Optional[str] = None,
@@ -454,8 +412,14 @@ async def alog_metric(
     if not key or not isinstance(key, str):
         raise ValueError("`key` must be a non-empty string")
 
-    if not value or not isinstance(value, (int, float)):
-        raise ValueError("`value` must be a scalar (int or float)")
+    if value is None:
+        raise ValueError("Must provide a value for the metric")
+
+    if not isinstance(value, (int, float)):
+        try:
+            value = float(value)
+        except (ValueError, TypeError):
+            raise ValueError("`value` must be a scalar (int or float)")
 
     try:
         return await _post(
@@ -489,7 +453,7 @@ def log_metric(
     Unit metrics are key-value pairs where the key is the metric name and the value is
     a scalar (int or float). These key-value pairs are associated with the currently
     selected model (inventory model in the ValidMind Platform) and keys can be logged
-    to over time to create a history of the metric. On the platform, these metrics
+    to over time to create a history of the metric. On the ValidMind Platform, these metrics
     will be used to create plots/visualizations for documentation and dashboards etc.
 
     Args: