validmind 2.2.6__py3-none-any.whl → 2.3.1__py3-none-any.whl

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.2.6"
+__version__ = "2.3.1"

validmind/{ai.py → ai/test_descriptions.py}

@@ -2,17 +2,22 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import concurrent.futures
 import os
+from concurrent.futures import ThreadPoolExecutor
 
-from openai import AzureOpenAI, OpenAI
+from validmind.utils import md_to_html
 
-from .logging import get_logger
+from ..logging import get_logger
+
+__executor = ThreadPoolExecutor()
 
 logger = get_logger(__name__)
 
 
-SYSTEM_PROMPT = """
+AI_REVISION_NAME = "Generated by ValidMind AI"
+DEFAULT_REVISION_NAME = "Default Description"
+
+SYSTEM_PROMPT = """  # noqa
 You are an expert data scientist and MRM specialist.
 You are tasked with analyzing the results of a quantitative test run on some model or dataset.
 Your goal is to create a test description that will act as part of the model documentation.
@@ -20,13 +25,14 @@ You will provide both the developer and other consumers of the documentation wit
 The overarching theme to maintain is MRM documentation.
 
 Examine the provided statistical test results and compose a description of the results.
-This will act as the description and interpretation of the result in the model documentation.
-It will be displayed alongside the test results table and figures.
+The results are either in the form of serialized tables or images of plots.
+Compose a description and interpretation of the result to accompany it in MRM documentation.
+It will be read by other data scientists and developers and by validators and stakeholders.
 
+Use valid Markdown syntax to format the response.
 Avoid long sentences and complex vocabulary.
 Avoid overly verbose explanations - the goal is to explain to a user what they are seeing in the results.
 Structure the response clearly and logically.
-Use valid Markdown syntax to format the response.
 Respond only with your analysis and insights, not the verbatim test results.
 Respond only with the markdown content, no explanation or context for your response is necessary.
 Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
@@ -61,9 +67,9 @@ Test ID: `{test_name}`
 {test_description}
 </Test Docstring>
 
-<Test Results Summary>
+<Test Results Table(s)>
 {test_summary}
-</Test Results Summary>
+</Test Results Table(s)>
 """.strip()
 
 
@@ -77,57 +83,6 @@ Test ID: `{test_name}`
 The attached plots show the results of the test.
 """.strip()
 
-__client = None
-__model = None
-
-# can be None, True or False (ternary to represent initial state, ack and failed ack)
-__ack = None
-
-__executor = concurrent.futures.ThreadPoolExecutor()
-
-
-def __get_client_and_model():
-    """Get model and client to use for generating interpretations
-
-    On first call, it will look in the environment for the API key endpoint, model etc.
-    and store them in a global variable to avoid loading them up again.
-    """
-    global __client, __model
-
-    if __client and __model:
-        return __client, __model
-
-    if "OPENAI_API_KEY" in os.environ:
-        __client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
-        __model = os.getenv("VM_OPENAI_MODEL", "gpt-4o")
-
-        logger.debug(f"Using OpenAI {__model} for generating descriptions")
-
-    elif "AZURE_OPENAI_KEY" in os.environ:
-        if "AZURE_OPENAI_ENDPOINT" not in os.environ:
-            raise ValueError(
-                "AZURE_OPENAI_ENDPOINT must be set to run LLM tests with Azure"
-            )
-
-        if "AZURE_OPENAI_MODEL" not in os.environ:
-            raise ValueError(
-                "AZURE_OPENAI_MODEL must be set to run LLM tests with Azure"
-            )
-
-        __client = AzureOpenAI(
-            azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"),
-            api_key=os.getenv("AZURE_OPENAI_KEY"),
-            api_version=os.getenv("AZURE_OPENAI_VERSION", "2023-05-15"),
-        )
-        __model = os.getenv("AZURE_OPENAI_MODEL")
-
-        logger.debug(f"Using Azure OpenAI {__model} for generating descriptions")
-
-    else:
-        raise ValueError("OPENAI_API_KEY or AZURE_OPENAI_KEY must be set")
-
-    return __client, __model
-
 
 class DescriptionFuture:
     """This will be immediately returned from generate_description so that
@@ -142,8 +97,6 @@ class DescriptionFuture:
         self._future = future
 
     def get_description(self):
-        from .utils import md_to_html
-
         if isinstance(self._future, str):
             description = self._future
         else:
@@ -163,7 +116,11 @@ def generate_description(
     if not test_summary and not figures:
         raise ValueError("No summary or figures provided - cannot generate description")
 
-    client, model = __get_client_and_model()
+    # TODO: fix circular import
+    from validmind.ai.utils import get_client_and_model
+
+    client, model = get_client_and_model()
+
     # get last part of test id
     test_name = test_id.split(".")[-1]
     # truncate the test description to save time
@@ -255,26 +212,61 @@ def background_generate_description(
     return DescriptionFuture(__executor.submit(wrapped))
 
 
-def is_configured():
-    global __ack
+def get_description_metadata(
+    test_id,
+    default_description,
+    summary=None,
+    figures=None,
+    prefix="metric_description",
+):
+    """Get Metadata Dictionary for a Test or Metric Result
+
+    Generates an LLM interpretation of the test results or uses the default
+    description and returns a metadata object that can be logged with the test results.
 
-    if __ack:
-        return True
+    By default, the description is generated by an LLM that will interpret the test
+    results and provide a human-readable description. If the summary or figures are
+    not provided, or the `VALIDMIND_LLM_DESCRIPTIONS_ENABLED` environment variable is
+    set to `0` or `false` or no LLM has been configured, the default description will
+    be used as the test result description.
 
-    try:
-        client, model = __get_client_and_model()
-        # send an empty message with max_tokens=1 to "ping" the API
-        response = client.chat.completions.create(
-            model=model,
-            messages=[{"role": "user", "content": ""}],
-            max_tokens=1,
-        )
-        logger.debug(
-            f"Received response from OpenAI: {response.choices[0].message.content}"
+    Note: Either the summary or figures must be provided to generate the description.
+
+    Args:
+        test_id (str): The test ID
+        default_description (str): The default description for the test
+        summary (Any): The test summary or results to interpret
+        figures (List[Figure]): The figures to attach to the test suite result
+        prefix (str): The prefix to use for the content ID (Default: "metric_description")
+
+    Returns:
+        dict: The metadata object to be logged with the test results
+    """
+    env_disabled = os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1") in [
+        "0",
+        "false",
+    ]
+
+    # TODO: fix circular import
+    from validmind.ai.utils import is_configured
+
+    if (summary or figures) and not env_disabled and is_configured():
+        revision_name = AI_REVISION_NAME
+
+        # get description future and set it as the description in the metadata
+        # this will lazily retrieved so it can run in the background in parallel
+        description = background_generate_description(
+            test_id=test_id,
+            test_description=default_description,
+            test_summary=summary,
+            figures=figures,
         )
-        __ack = True
-    except Exception as e:
-        logger.debug(f"Failed to connect to OpenAI: {e}")
-        __ack = False
 
-    return __ack
+    else:
+        revision_name = DEFAULT_REVISION_NAME
+        description = md_to_html(default_description, mathml=True)
+
+    return {
+        "content_id": f"{prefix}:{test_id}::{revision_name}",
+        "text": description,
+    }

validmind/ai/utils.py

@@ -0,0 +1,104 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import os
+
+from openai import AzureOpenAI, Client, OpenAI
+
+from ..api_client import get_ai_key, get_api_host
+from ..logging import get_logger
+
+logger = get_logger(__name__)
+
+
+__client = None
+__model = None
+# can be None, True or False (ternary to represent initial state, ack and failed ack)
+__ack = None
+
+
+def get_client_and_model():
+    """Get model and client to use for generating interpretations
+
+    On first call, it will look in the environment for the API key endpoint, model etc.
+    and store them in a global variable to avoid loading them up again.
+    """
+    global __client, __model
+
+    if __client and __model:
+        return __client, __model
+
+    if "OPENAI_API_KEY" in os.environ:
+        __client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+        __model = os.getenv("VM_OPENAI_MODEL", "gpt-4o")
+
+        logger.debug(f"Using OpenAI {__model} for generating descriptions")
+
+    elif "AZURE_OPENAI_KEY" in os.environ:
+        if "AZURE_OPENAI_ENDPOINT" not in os.environ:
+            raise ValueError(
+                "AZURE_OPENAI_ENDPOINT must be set to run LLM tests with Azure"
+            )
+
+        if "AZURE_OPENAI_MODEL" not in os.environ:
+            raise ValueError(
+                "AZURE_OPENAI_MODEL must be set to run LLM tests with Azure"
+            )
+
+        __client = AzureOpenAI(
+            azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"),
+            api_key=os.getenv("AZURE_OPENAI_KEY"),
+            api_version=os.getenv("AZURE_OPENAI_VERSION", "2023-05-15"),
+        )
+        __model = os.getenv("AZURE_OPENAI_MODEL")
+
+        logger.debug(f"Using Azure OpenAI {__model} for generating descriptions")
+
+    else:
+        try:
+            response = get_ai_key()
+            __client = Client(
+                base_url=(
+                    # TODO: improve this to be a bit more dynamic
+                    "http://localhost:4000/genai"
+                    if "localhost" in get_api_host()
+                    else f"{get_api_host()}/genai"
+                ),
+                api_key=response["key"],
+            )
+            __model = "gpt-4o"  # TODO: backend should tell us which model to use
+            logger.debug(f"Using ValidMind {__model} for generating descriptions")
+        except Exception as e:
+            logger.debug(f"Failed to get API key: {e}")
+            raise ValueError(
+                "OPENAI_API_KEY, AZURE_OPENAI_KEY must be set, or your account "
+                "must be setup to use ValidMind's LLM in order to use LLM features"
+            )
+
+    return __client, __model
+
+
+def is_configured():
+    global __ack
+
+    if __ack:
+        return True
+
+    try:
+        client, model = get_client_and_model()
+        # send an empty message with max_tokens=1 to "ping" the API
+        response = client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": ""}],
+            max_tokens=1,
+        )
+        logger.debug(
+            f"Received response from OpenAI: {response.choices[0].message.content}"
+        )
+        __ack = True
+    except Exception as e:
+        logger.debug(f"Failed to connect to OpenAI: {e}")
+        __ack = False
+
+    return __ack

validmind/api_client.py

@@ -11,9 +11,9 @@ import asyncio
 import atexit
 import json
 import os
-import urllib.parse
 from io import BytesIO
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from urllib.parse import urlencode, urljoin
 
 import aiohttp
 import requests
@@ -69,6 +69,14 @@ def get_api_project() -> Optional[str]:
     return _project
 
 
+def get_api_headers() -> Dict[str, str]:
+    return {
+        "X-API-KEY": _api_key,
+        "X-API-SECRET": _api_secret,
+        "X-PROJECT-CUID": _project,
+    }
+
+
 def init(
     project: Optional[str] = None,
     api_key: Optional[str] = None,
@@ -97,10 +105,7 @@ def init(
         # special case to detect when running a notebook with the standard init snippet
         # will override with environment variables so we don't have to keep updating
         # the notebook
-        api_host = None
-        api_key = None
-        api_secret = None
-        project = None
+        api_host = api_key = api_secret = project = None
 
     _project = project or os.getenv("VM_API_PROJECT")
 
@@ -114,8 +119,9 @@ def init(
         raise MissingAPICredentialsError()
 
     _api_host = api_host or os.getenv(
-        "VM_API_HOST", "http://127.0.0.1:5000/api/v1/tracking"
+        "VM_API_HOST", "http://127.0.0.1:5000/api/v1/tracking/"
     )
+
     _run_cuid = os.getenv("VM_RUN_CUID", None)
 
     try:
@@ -127,7 +133,7 @@ def init(
         raise e
 
 
-async def _get_session() -> aiohttp.ClientSession:
+def _get_session() -> aiohttp.ClientSession:
     """Initializes the async client session"""
     global __api_session
 
@@ -147,7 +153,7 @@ async def _get_session() -> aiohttp.ClientSession:
 def __ping() -> Dict[str, Any]:
     """Validates that we can connect to the ValidMind API (does not use the async session)"""
     r = requests.get(
-        f"{_api_host}/ping",
+        __get_url("ping", should_start_run=False),
         headers={
             "X-API-KEY": _api_key,
             "X-API-SECRET": _api_secret,
@@ -189,21 +195,35 @@ def reload():
         raise e
 
 
-async def __get_url(endpoint: str, params: Optional[Dict[str, str]] = None) -> str:
-    if not _run_cuid:
-        start_run()
+def __get_url(
+    endpoint: str,
+    params: Optional[Dict[str, str]] = None,
+    should_start_run: bool = True,
+) -> str:
+    global _api_host
 
     params = params or {}
-    params["run_cuid"] = _run_cuid
 
-    return f"{_api_host}/{endpoint}?{urllib.parse.urlencode(params)}"
+    if not _run_cuid and should_start_run:
+        start_run()
+
+    if should_start_run:
+        params["run_cuid"] = _run_cuid
+
+    if not _api_host.endswith("/"):
+        _api_host += "/"
+
+    if params:
+        return f"{urljoin(_api_host, endpoint)}?{urlencode(params)}"
+
+    return urljoin(_api_host, endpoint)
 
 
 async def _get(
     endpoint: str, params: Optional[Dict[str, str]] = None
 ) -> Dict[str, Any]:
-    url = await __get_url(endpoint, params)
-    session = await _get_session()
+    url = __get_url(endpoint, params)
+    session = _get_session()
     session.headers.update({"X-RUN-CUID": _run_cuid})
 
     async with session.get(url) as r:
@@ -219,8 +239,8 @@ async def _post(
     data: Optional[Union[dict, FormData]] = None,
     files: Optional[Dict[str, Tuple[str, BytesIO, str]]] = None,
 ) -> Dict[str, Any]:
-    url = await __get_url(endpoint, params)
-    session = await _get_session()
+    url = __get_url(endpoint, params)
+    session = _get_session()
     session.headers.update({"X-RUN-CUID": _run_cuid})
 
     if not isinstance(data, (dict)) and files is not None:
@@ -491,7 +511,7 @@ def log_test_results(
     return responses
 
 
-def _log_input(name: str, type: str, metadata: Dict[str, Any]) -> Dict[str, Any]:
+def log_input(name: str, type: str, metadata: Dict[str, Any]) -> Dict[str, Any]:
     """Logs input information - internal use for now (don't expose via public API)
 
     Args:
@@ -539,7 +559,7 @@ def start_run() -> str:
     global _run_cuid
 
     r = requests.post(
-        f"{_api_host}/start_run",
+        __get_url("start_run", should_start_run=False),
         headers={
             "X-API-KEY": _api_key,
             "X-API-SECRET": _api_secret,
@@ -555,3 +575,22 @@ def start_run() -> str:
     _run_cuid = test_run["cuid"]
 
     return test_run["cuid"]
+
+
+def get_ai_key() -> str:
+    """Calls the api to get an api key for our LLM proxy"""
+    r = requests.get(
+        __get_url("ai/key", should_start_run=False),
+        headers={
+            "X-API-KEY": _api_key,
+            "X-API-SECRET": _api_secret,
+            "X-PROJECT-CUID": _project,
+        },
+    )
+
+    if r.status_code != 200:
+        # TODO: improve error handling when there's no Open AI API or AI key available
+        # logger.error("Could not get AI key from ValidMind API")
+        raise_api_error(r.text)
+
+    return r.json()

validmind/client.py

@@ -9,7 +9,7 @@ Client interface for all data and model validation functions
 import pandas as pd
 import polars as pl
 
-from .api_client import _log_input as log_input
+from .api_client import log_input as log_input
 from .client_config import client_config
 from .errors import (
     GetTestSuiteError,
@@ -180,6 +180,7 @@ def init_model(
     attributes: dict = None,
     predict_fn: callable = None,
     __log=True,
+    **kwargs,
 ) -> VMModel:
     """
     Initializes a VM Model, which can then be passed to other functions
@@ -194,6 +195,7 @@ def init_model(
             this to the same key.
         attributes (dict): A dictionary of model attributes
         predict_fn (callable): A function that takes an input and returns a prediction
+        **kwargs: Additional arguments to pass to the model
 
     Raises:
         ValueError: If the model type is not supported
@@ -246,6 +248,7 @@ def init_model(
             input_id=input_id,
             model=model,  # Trained model instance
             predict_fn=predict_fn,
+            **kwargs,
         )
         metadata = get_model_info(vm_model)
     else:
@@ -351,10 +354,7 @@ def get_test_suite(
             )
 
         return get_template_test_suite(
-            client_config.documentation_template,
-            section=section,
-            *args,
-            **kwargs,
+            client_config.documentation_template, section=section
         )
 
     return get_test_suite_by_id(test_suite_id)(*args, **kwargs)

validmind/models/foundation.py

@@ -15,7 +15,7 @@ logger = get_logger(__name__)
 @dataclass
 class Prompt:
     template: str
-    variables: list
+    variables: list = None
 
 
 class FoundationModel(FunctionModel):
@@ -33,17 +33,21 @@ class FoundationModel(FunctionModel):
     """
 
     def __post_init__(self):
-        if not getattr(self, "predict_fn") or not callable(self.predict_fn):
-            raise ValueError("FoundationModel requires a callable predict_fn")
+        super().__post_init__()
 
-        self.name = self.name or self.predict_fn.__name__
+        if not hasattr(self, "prompt") or not isinstance(self.prompt, Prompt):
+            raise ValueError("FoundationModel requires a Prompt object")
 
     def _build_prompt(self, x: pd.DataFrame):
         """
         Builds the prompt for the model
         """
-        return self.prompt.template.format(
-            **{key: x[key] for key in self.prompt.variables}
+        return (
+            self.prompt.template.format(
+                **{key: x[key] for key in self.prompt.variables}
+            )
+            if self.prompt.variables
+            else self.prompt.template
         )
 
     def predict(self, X: pd.DataFrame):

validmind/models/function.py

@@ -31,10 +31,12 @@ class FunctionModel(VMModel):
             input features and return a prediction.
         input_id (str, optional): The input ID for the model. Defaults to None.
         name (str, optional): The name of the model. Defaults to the name of the predict_fn.
+        prompt (Prompt, optional): If using a prompt, the prompt object that defines the template
+            and the variables (if any). Defaults to None.
     """
 
     def __post_init__(self):
-        if not getattr(self, "predict_fn") or not callable(self.predict_fn):
+        if not hasattr(self, "predict_fn") or not callable(self.predict_fn):
             raise ValueError("FunctionModel requires a callable predict_fn")
 
         self.name = self.name or self.predict_fn.__name__

validmind/models/metadata.py

@@ -24,7 +24,7 @@ class MetadataModel(VMModel):
     """
 
     def __post_init__(self):
-        if not getattr(self, "attributes"):
+        if not hasattr(self, "attributes"):
             raise ValueError("MetadataModel requires attributes")
 
         self.name = self.name or "Metadata Model"

validmind/test_suites/__init__.py

@@ -25,12 +25,7 @@ from .embeddings import EmbeddingsFullSuite, EmbeddingsMetrics, EmbeddingsPerfor
 from .llm import LLMClassifierFullSuite, PromptValidation
 from .nlp import NLPClassifierFullSuite
 from .parameters_optimization import KmeansParametersOptimization
-from .regression import (
-    RegressionFullSuite,
-    RegressionMetrics,
-    RegressionModelsComparison,
-    RegressionPerformance,
-)
+from .regression import RegressionFullSuite, RegressionMetrics, RegressionPerformance
 from .statsmodels_timeseries import (
     RegressionModelDescription,
     RegressionModelsEvaluation,
@@ -72,7 +67,6 @@ core_test_suites = {
     RegressionMetrics.suite_id: RegressionMetrics,
     RegressionModelDescription.suite_id: RegressionModelDescription,
     RegressionModelsEvaluation.suite_id: RegressionModelsEvaluation,
-    RegressionModelsComparison.suite_id: RegressionModelsComparison,
     RegressionFullSuite.suite_id: RegressionFullSuite,
     RegressionPerformance.suite_id: RegressionPerformance,
     SummarizationMetrics.suite_id: SummarizationMetrics,

validmind/test_suites/regression.py

@@ -32,17 +32,6 @@ class RegressionPerformance(TestSuite):
     ]
 
 
-class RegressionModelsComparison(TestSuite):
-    """
-    Test suite for regression models performance comparison
-    """
-
-    suite_id = "regression_models_comparison"
-    tests = [
-        "validmind.model_validation.sklearn.RegressionModelsPerformanceComparison",
-    ]
-
-
 class RegressionFullSuite(TestSuite):
     """
     Full test suite for regression models.
@@ -70,9 +59,4 @@ class RegressionFullSuite(TestSuite):
             "section_description": RegressionPerformance.__doc__,
             "section_tests": RegressionPerformance.tests,
         },
-        {
-            "section_id": RegressionModelsComparison.suite_id,
-            "section_description": RegressionModelsComparison.__doc__,
-            "section_tests": RegressionModelsComparison.tests,
-        },
     ]

validmind/test_suites/statsmodels_timeseries.py

@@ -29,5 +29,5 @@ class RegressionModelsEvaluation(TestSuite):
     suite_id = "regression_models_evaluation"
     tests = [
         "validmind.model_validation.statsmodels.RegressionModelsCoeffs",
-        "validmind.model_validation.statsmodels.RegressionModelsPerformance",
+        "validmind.model_validation.sklearn.RegressionModelsPerformanceComparison",
     ]