validmind 2.2.5__tar.gz → 2.3.1__tar.gz

{validmind-2.2.5 → validmind-2.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.2.5
+Version: 2.3.1
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez

{validmind-2.2.5 → validmind-2.3.1}/pyproject.toml

@@ -10,7 +10,7 @@ description = "ValidMind Developer Framework"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.2.5"
+version = "2.3.1"
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<3.12"

validmind-2.3.1/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.3.1"

validmind-2.2.5/validmind/ai.py → validmind-2.3.1/validmind/ai/test_descriptions.py

@@ -2,12 +2,22 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import concurrent.futures
 import os
+from concurrent.futures import ThreadPoolExecutor
 
-from openai import AzureOpenAI, OpenAI
+from validmind.utils import md_to_html
 
-SYSTEM_PROMPT = """
+from ..logging import get_logger
+
+__executor = ThreadPoolExecutor()
+
+logger = get_logger(__name__)
+
+
+AI_REVISION_NAME = "Generated by ValidMind AI"
+DEFAULT_REVISION_NAME = "Default Description"
+
+SYSTEM_PROMPT = """  # noqa
 You are an expert data scientist and MRM specialist.
 You are tasked with analyzing the results of a quantitative test run on some model or dataset.
 Your goal is to create a test description that will act as part of the model documentation.
@@ -15,12 +25,14 @@ You will provide both the developer and other consumers of the documentation wit
 The overarching theme to maintain is MRM documentation.
 
 Examine the provided statistical test results and compose a description of the results.
-This will act as the description and interpretation of the result in the model documentation.
-It will be displayed alongside the test results table and figures.
+The results are either in the form of serialized tables or images of plots.
+Compose a description and interpretation of the result to accompany it in MRM documentation.
+It will be read by other data scientists and developers and by validators and stakeholders.
 
+Use valid Markdown syntax to format the response.
 Avoid long sentences and complex vocabulary.
+Avoid overly verbose explanations - the goal is to explain to a user what they are seeing in the results.
 Structure the response clearly and logically.
-Use valid Markdown syntax to format the response.
 Respond only with your analysis and insights, not the verbatim test results.
 Respond only with the markdown content, no explanation or context for your response is necessary.
 Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
@@ -28,9 +40,10 @@ Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" ->
 Explain the test, its purpose, its mechanism/formula etc and why it is useful.
 If relevant, provide a very brief description of the way this test is used in model/dataset evaluation and how it is interpreted.
 Highlight the key insights from the test results. The key insights should be concise and easily understood.
+An insight should only be included if it is something not entirely obvious from the test results.
 End the response with any closing remarks, summary or additional useful information.
 
-Use the following format for the response (feel free to modify slightly if necessary):
+Use the following format for the response (feel free to stray from it if necessary - this is a suggested starting point):
 
 <ResponseFormat>
 **<Test Name>** calculates the xyz <continue to explain what it does in detail>...
@@ -54,9 +67,9 @@ Test ID: `{test_name}`
 {test_description}
 </Test Docstring>
 
-<Test Results Summary>
+<Test Results Table(s)>
 {test_summary}
-</Test Results Summary>
+</Test Results Table(s)>
 """.strip()
 
 
@@ -70,48 +83,6 @@ Test ID: `{test_name}`
 The attached plots show the results of the test.
 """.strip()
 
-__client = None
-__model = None
-
-__executor = concurrent.futures.ThreadPoolExecutor()
-
-
-def __get_client_and_model():
-    """
-    Get the model to use for generating interpretations
-    """
-    global __client, __model
-
-    if __client and __model:
-        return __client, __model
-
-    if "OPENAI_API_KEY" in os.environ:
-        __client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
-        __model = os.environ.get("VM_OPENAI_MODEL", "gpt-4o")
-
-    elif "AZURE_OPENAI_KEY" in os.environ:
-        if "AZURE_OPENAI_ENDPOINT" not in os.environ:
-            raise ValueError(
-                "AZURE_OPENAI_ENDPOINT must be set to run LLM tests with Azure"
-            )
-
-        if "AZURE_OPENAI_MODEL" not in os.environ:
-            raise ValueError(
-                "AZURE_OPENAI_MODEL must be set to run LLM tests with Azure"
-            )
-
-        __client = AzureOpenAI(
-            azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"),
-            api_key=os.environ.get("AZURE_OPENAI_KEY"),
-            api_version=os.environ.get("AZURE_OPENAI_VERSION", "2023-05-15"),
-        )
-        __model = os.environ.get("AZURE_OPENAI_MODEL")
-
-    else:
-        raise ValueError("OPENAI_API_KEY or AZURE_OPENAI_KEY must be set")
-
-    return __client, __model
-
 
 class DescriptionFuture:
     """This will be immediately returned from generate_description so that
@@ -126,12 +97,17 @@ class DescriptionFuture:
         self._future = future
 
     def get_description(self):
-        # This will block until the future is completed
-        return self._future.result()
+        if isinstance(self._future, str):
+            description = self._future
+        else:
+            # This will block until the future is completed
+            description = self._future.result()
 
+        return md_to_html(description, mathml=True)
 
-def generate_description_async(
-    test_name: str,
+
+def generate_description(
+    test_id: str,
     test_description: str,
     test_summary: str,
     figures: list = None,
@@ -140,14 +116,29 @@ def generate_description_async(
     if not test_summary and not figures:
         raise ValueError("No summary or figures provided - cannot generate description")
 
-    client, _ = __get_client_and_model()
+    # TODO: fix circular import
+    from validmind.ai.utils import get_client_and_model
+
+    client, model = get_client_and_model()
+
     # get last part of test id
-    test_name = test_name.split(".")[-1]
+    test_name = test_id.split(".")[-1]
+    # truncate the test description to save time
+    test_description = (
+        f"{test_description[:500]}..."
+        if len(test_description) > 500
+        else test_description
+    )
 
     if test_summary:
+        logger.debug(
+            f"Generating description for test {test_name} with stringified summary"
+        )
         return (
             client.chat.completions.create(
-                model="gpt-4o",
+                model=model,
+                temperature=0,
+                seed=42,
                 messages=[
                     {"role": "system", "content": SYSTEM_PROMPT},
                     {
@@ -164,9 +155,14 @@ def generate_description_async(
             .message.content.strip()
         )
 
+    logger.debug(
+        f"Generating description for test {test_name} with {len(figures)} figures"
+    )
     return (
         client.chat.completions.create(
-            model="gpt-4o",
+            model=model,
+            temperature=0,
+            seed=42,
             messages=[
                 {"role": "system", "content": SYSTEM_PROMPT},
                 {
@@ -197,18 +193,80 @@ def generate_description_async(
     )
 
 
-def generate_description(
-    test_name: str,
+def background_generate_description(
+    test_id: str,
     test_description: str,
     test_summary: str,
     figures: list = None,
 ):
-    future = __executor.submit(
-        generate_description_async,
-        test_name,
-        test_description,
-        test_summary,
-        figures,
-    )
+    def wrapped():
+        try:
+            return generate_description(
+                test_id, test_description, test_summary, figures
+            )
+        except Exception as e:
+            logger.error(f"Failed to generate description: {e}")
+
+            return test_description
+
+    return DescriptionFuture(__executor.submit(wrapped))
+
+
+def get_description_metadata(
+    test_id,
+    default_description,
+    summary=None,
+    figures=None,
+    prefix="metric_description",
+):
+    """Get Metadata Dictionary for a Test or Metric Result
+
+    Generates an LLM interpretation of the test results or uses the default
+    description and returns a metadata object that can be logged with the test results.
+
+    By default, the description is generated by an LLM that will interpret the test
+    results and provide a human-readable description. If the summary or figures are
+    not provided, or the `VALIDMIND_LLM_DESCRIPTIONS_ENABLED` environment variable is
+    set to `0` or `false` or no LLM has been configured, the default description will
+    be used as the test result description.
+
+    Note: Either the summary or figures must be provided to generate the description.
+
+    Args:
+        test_id (str): The test ID
+        default_description (str): The default description for the test
+        summary (Any): The test summary or results to interpret
+        figures (List[Figure]): The figures to attach to the test suite result
+        prefix (str): The prefix to use for the content ID (Default: "metric_description")
+
+    Returns:
+        dict: The metadata object to be logged with the test results
+    """
+    env_disabled = os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1") in [
+        "0",
+        "false",
+    ]
+
+    # TODO: fix circular import
+    from validmind.ai.utils import is_configured
+
+    if (summary or figures) and not env_disabled and is_configured():
+        revision_name = AI_REVISION_NAME
+
+        # get description future and set it as the description in the metadata
+        # this will lazily retrieved so it can run in the background in parallel
+        description = background_generate_description(
+            test_id=test_id,
+            test_description=default_description,
+            test_summary=summary,
+            figures=figures,
+        )
+
+    else:
+        revision_name = DEFAULT_REVISION_NAME
+        description = md_to_html(default_description, mathml=True)
 
-    return DescriptionFuture(future)
+    return {
+        "content_id": f"{prefix}:{test_id}::{revision_name}",
+        "text": description,
+    }

validmind-2.3.1/validmind/ai/utils.py

@@ -0,0 +1,104 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import os
+
+from openai import AzureOpenAI, Client, OpenAI
+
+from ..api_client import get_ai_key, get_api_host
+from ..logging import get_logger
+
+logger = get_logger(__name__)
+
+
+__client = None
+__model = None
+# can be None, True or False (ternary to represent initial state, ack and failed ack)
+__ack = None
+
+
+def get_client_and_model():
+    """Get model and client to use for generating interpretations
+
+    On first call, it will look in the environment for the API key endpoint, model etc.
+    and store them in a global variable to avoid loading them up again.
+    """
+    global __client, __model
+
+    if __client and __model:
+        return __client, __model
+
+    if "OPENAI_API_KEY" in os.environ:
+        __client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+        __model = os.getenv("VM_OPENAI_MODEL", "gpt-4o")
+
+        logger.debug(f"Using OpenAI {__model} for generating descriptions")
+
+    elif "AZURE_OPENAI_KEY" in os.environ:
+        if "AZURE_OPENAI_ENDPOINT" not in os.environ:
+            raise ValueError(
+                "AZURE_OPENAI_ENDPOINT must be set to run LLM tests with Azure"
+            )
+
+        if "AZURE_OPENAI_MODEL" not in os.environ:
+            raise ValueError(
+                "AZURE_OPENAI_MODEL must be set to run LLM tests with Azure"
+            )
+
+        __client = AzureOpenAI(
+            azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"),
+            api_key=os.getenv("AZURE_OPENAI_KEY"),
+            api_version=os.getenv("AZURE_OPENAI_VERSION", "2023-05-15"),
+        )
+        __model = os.getenv("AZURE_OPENAI_MODEL")
+
+        logger.debug(f"Using Azure OpenAI {__model} for generating descriptions")
+
+    else:
+        try:
+            response = get_ai_key()
+            __client = Client(
+                base_url=(
+                    # TODO: improve this to be a bit more dynamic
+                    "http://localhost:4000/genai"
+                    if "localhost" in get_api_host()
+                    else f"{get_api_host()}/genai"
+                ),
+                api_key=response["key"],
+            )
+            __model = "gpt-4o"  # TODO: backend should tell us which model to use
+            logger.debug(f"Using ValidMind {__model} for generating descriptions")
+        except Exception as e:
+            logger.debug(f"Failed to get API key: {e}")
+            raise ValueError(
+                "OPENAI_API_KEY, AZURE_OPENAI_KEY must be set, or your account "
+                "must be setup to use ValidMind's LLM in order to use LLM features"
+            )
+
+    return __client, __model
+
+
+def is_configured():
+    global __ack
+
+    if __ack:
+        return True
+
+    try:
+        client, model = get_client_and_model()
+        # send an empty message with max_tokens=1 to "ping" the API
+        response = client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": ""}],
+            max_tokens=1,
+        )
+        logger.debug(
+            f"Received response from OpenAI: {response.choices[0].message.content}"
+        )
+        __ack = True
+    except Exception as e:
+        logger.debug(f"Failed to connect to OpenAI: {e}")
+        __ack = False
+
+    return __ack

{validmind-2.2.5 → validmind-2.3.1}/validmind/api_client.py

@@ -11,9 +11,9 @@ import asyncio
 import atexit
 import json
 import os
-import urllib.parse
 from io import BytesIO
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from urllib.parse import urlencode, urljoin
 
 import aiohttp
 import requests
@@ -22,19 +22,19 @@ from aiohttp import FormData
 from .client_config import client_config
 from .errors import MissingAPICredentialsError, MissingProjectIdError, raise_api_error
 from .logging import get_logger, init_sentry, send_single_error
-from .utils import NumpyEncoder, md_to_html, run_async
+from .utils import NumpyEncoder, run_async
 from .vm_models import Figure, MetricResult, ThresholdTestResults
 
 # TODO: can't import types from vm_models because of circular dependency
 
 logger = get_logger(__name__)
 
-_api_key = os.environ.get("VM_API_KEY")
-_api_secret = os.environ.get("VM_API_SECRET")
-_api_host = os.environ.get("VM_API_HOST")
+_api_key = os.getenv("VM_API_KEY")
+_api_secret = os.getenv("VM_API_SECRET")
+_api_host = os.getenv("VM_API_HOST")
 
-_project = os.environ.get("VM_API_PROJECT")
-_run_cuid = os.environ.get("VM_RUN_CUID")
+_project = os.getenv("VM_API_PROJECT")
+_run_cuid = os.getenv("VM_RUN_CUID")
 
 __api_session: aiohttp.ClientSession = None
 
@@ -69,6 +69,14 @@ def get_api_project() -> Optional[str]:
     return _project
 
 
+def get_api_headers() -> Dict[str, str]:
+    return {
+        "X-API-KEY": _api_key,
+        "X-API-SECRET": _api_secret,
+        "X-PROJECT-CUID": _project,
+    }
+
+
 def init(
     project: Optional[str] = None,
     api_key: Optional[str] = None,
@@ -97,26 +105,24 @@ def init(
         # special case to detect when running a notebook with the standard init snippet
         # will override with environment variables so we don't have to keep updating
         # the notebook
-        api_host = None
-        api_key = None
-        api_secret = None
-        project = None
+        api_host = api_key = api_secret = project = None
 
-    _project = project or os.environ.get("VM_API_PROJECT")
+    _project = project or os.getenv("VM_API_PROJECT")
 
     if _project is None:
         raise MissingProjectIdError()
 
-    _api_key = api_key or os.environ.get("VM_API_KEY")
-    _api_secret = api_secret or os.environ.get("VM_API_SECRET")
+    _api_key = api_key or os.getenv("VM_API_KEY")
+    _api_secret = api_secret or os.getenv("VM_API_SECRET")
 
     if _api_key is None or _api_secret is None:
         raise MissingAPICredentialsError()
 
-    _api_host = api_host or os.environ.get(
-        "VM_API_HOST", "http://127.0.0.1:5000/api/v1/tracking"
+    _api_host = api_host or os.getenv(
+        "VM_API_HOST", "http://127.0.0.1:5000/api/v1/tracking/"
     )
-    _run_cuid = os.environ.get("VM_RUN_CUID", None)
+
+    _run_cuid = os.getenv("VM_RUN_CUID", None)
 
     try:
         __ping()
@@ -127,7 +133,7 @@ def init(
         raise e
 
 
-async def _get_session() -> aiohttp.ClientSession:
+def _get_session() -> aiohttp.ClientSession:
     """Initializes the async client session"""
     global __api_session
 
@@ -147,7 +153,7 @@ async def _get_session() -> aiohttp.ClientSession:
 def __ping() -> Dict[str, Any]:
     """Validates that we can connect to the ValidMind API (does not use the async session)"""
     r = requests.get(
-        f"{_api_host}/ping",
+        __get_url("ping", should_start_run=False),
         headers={
             "X-API-KEY": _api_key,
             "X-API-SECRET": _api_secret,
@@ -189,21 +195,35 @@ def reload():
         raise e
 
 
-async def __get_url(endpoint: str, params: Optional[Dict[str, str]] = None) -> str:
-    if not _run_cuid:
-        start_run()
+def __get_url(
+    endpoint: str,
+    params: Optional[Dict[str, str]] = None,
+    should_start_run: bool = True,
+) -> str:
+    global _api_host
 
     params = params or {}
-    params["run_cuid"] = _run_cuid
 
-    return f"{_api_host}/{endpoint}?{urllib.parse.urlencode(params)}"
+    if not _run_cuid and should_start_run:
+        start_run()
+
+    if should_start_run:
+        params["run_cuid"] = _run_cuid
+
+    if not _api_host.endswith("/"):
+        _api_host += "/"
+
+    if params:
+        return f"{urljoin(_api_host, endpoint)}?{urlencode(params)}"
+
+    return urljoin(_api_host, endpoint)
 
 
 async def _get(
     endpoint: str, params: Optional[Dict[str, str]] = None
 ) -> Dict[str, Any]:
-    url = await __get_url(endpoint, params)
-    session = await _get_session()
+    url = __get_url(endpoint, params)
+    session = _get_session()
     session.headers.update({"X-RUN-CUID": _run_cuid})
 
     async with session.get(url) as r:
@@ -219,8 +239,8 @@ async def _post(
     data: Optional[Union[dict, FormData]] = None,
     files: Optional[Dict[str, Tuple[str, BytesIO, str]]] = None,
 ) -> Dict[str, Any]:
-    url = await __get_url(endpoint, params)
-    session = await _get_session()
+    url = __get_url(endpoint, params)
+    session = _get_session()
     session.headers.update({"X-RUN-CUID": _run_cuid})
 
     if not isinstance(data, (dict)) and files is not None:
@@ -349,7 +369,7 @@ async def log_metadata(
     """
     metadata_dict = {"content_id": content_id}
     if text is not None:
-        metadata_dict["text"] = md_to_html(text, mathml=True)
+        metadata_dict["text"] = text
     if _json is not None:
         metadata_dict["json"] = _json
 
@@ -491,7 +511,7 @@ def log_test_results(
     return responses
 
 
-def _log_input(name: str, type: str, metadata: Dict[str, Any]) -> Dict[str, Any]:
+def log_input(name: str, type: str, metadata: Dict[str, Any]) -> Dict[str, Any]:
     """Logs input information - internal use for now (don't expose via public API)
 
     Args:
@@ -539,7 +559,7 @@ def start_run() -> str:
     global _run_cuid
 
     r = requests.post(
-        f"{_api_host}/start_run",
+        __get_url("start_run", should_start_run=False),
         headers={
             "X-API-KEY": _api_key,
             "X-API-SECRET": _api_secret,
@@ -555,3 +575,22 @@ def start_run() -> str:
     _run_cuid = test_run["cuid"]
 
     return test_run["cuid"]
+
+
+def get_ai_key() -> str:
+    """Calls the api to get an api key for our LLM proxy"""
+    r = requests.get(
+        __get_url("ai/key", should_start_run=False),
+        headers={
+            "X-API-KEY": _api_key,
+            "X-API-SECRET": _api_secret,
+            "X-PROJECT-CUID": _project,
+        },
+    )
+
+    if r.status_code != 200:
+        # TODO: improve error handling when there's no Open AI API or AI key available
+        # logger.error("Could not get AI key from ValidMind API")
+        raise_api_error(r.text)
+
+    return r.json()

{validmind-2.2.5 → validmind-2.3.1}/validmind/client.py

@@ -9,7 +9,7 @@ Client interface for all data and model validation functions
 import pandas as pd
 import polars as pl
 
-from .api_client import _log_input as log_input
+from .api_client import log_input as log_input
 from .client_config import client_config
 from .errors import (
     GetTestSuiteError,
@@ -180,6 +180,7 @@ def init_model(
     attributes: dict = None,
     predict_fn: callable = None,
     __log=True,
+    **kwargs,
 ) -> VMModel:
     """
     Initializes a VM Model, which can then be passed to other functions
@@ -194,6 +195,7 @@ def init_model(
             this to the same key.
         attributes (dict): A dictionary of model attributes
         predict_fn (callable): A function that takes an input and returns a prediction
+        **kwargs: Additional arguments to pass to the model
 
     Raises:
         ValueError: If the model type is not supported
@@ -246,6 +248,7 @@ def init_model(
             input_id=input_id,
             model=model,  # Trained model instance
             predict_fn=predict_fn,
+            **kwargs,
         )
         metadata = get_model_info(vm_model)
     else:
@@ -351,10 +354,7 @@ def get_test_suite(
             )
 
         return get_template_test_suite(
-            client_config.documentation_template,
-            section=section,
-            *args,
-            **kwargs,
+            client_config.documentation_template, section=section
         )
 
     return get_test_suite_by_id(test_suite_id)(*args, **kwargs)