validmind 2.5.19__py3-none-any.whl → 2.5.24__py3-none-any.whl

validmind/__init__.py

@@ -3,14 +3,14 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-ValidMind’s Python Developer Framework is a library of developer tools and methods designed to automate
+ValidMind’s Python Library is a library of developer tools and methods designed to automate
 the documentation and validation of your models.
 
-The Developer Framework is designed to be model agnostic. If your model is built in Python, ValidMind's
+The Library is designed to be model agnostic. If your model is built in Python, ValidMind's
 Python library will provide all the standard functionality without requiring your developers to rewrite any functions.
 
-The Developer Framework provides a rich suite of documentation tools and test suites, from documenting
-descriptions of your dataset to testing your models for weak spots and overfit areas. The Developer Framework
+The Library provides a rich suite of documentation tools and test suites, from documenting
+descriptions of your dataset to testing your models for weak spots and overfit areas. The Library
 helps you automate the generation of model documentation by feeding the ValidMind platform with documentation
 artifacts and test results to the ValidMind platform.
 
@@ -35,7 +35,7 @@ vm.init(
 ```
 
 After you have pasted the code snippet into your development source code and executed the code, the Python client
-library will register with ValidMind. You can now use the Developer Framework to document and test your models,
+library will register with ValidMind. You can now use the Library to document and test your models,
 and to upload to the ValidMind Platform.
 """
 import warnings
@@ -47,10 +47,7 @@ warnings.simplefilter("ignore", category=NumbaDeprecationWarning)
 warnings.simplefilter("ignore", category=NumbaPendingDeprecationWarning)
 
 from .__version__ import __version__  # noqa: E402
-from .api_client import init
-from .api_client import log_figure as _log_figure_async
-from .api_client import log_metrics as _log_metrics_async
-from .api_client import log_test_results, reload
+from .api_client import init, log_metric, reload
 from .client import (  # noqa: E402
     get_test_suite,
     init_dataset,
@@ -61,39 +58,6 @@ from .client import (  # noqa: E402
     run_test_suite,
 )
 from .tests.decorator import metric, tags, tasks, test
-from .utils import run_async  # noqa: E402
-
-
-def log_metrics(metrics, inputs=None):
-    """Logs metrics to ValidMind API.
-
-    Args:
-        metrics (list): A list of Metric objects
-        inputs (list): A list of input names to associate with the metrics
-
-    Raises:
-        Exception: If the API call fails
-
-    Returns:
-        dict: The response from the API
-    """
-    run_async(_log_metrics_async, metrics, inputs)
-
-
-def log_figure(figure):
-    """Logs a figure
-
-    Args:
-        figure (Figure): The Figure object wrapper
-
-    Raises:
-        Exception: If the API call fails
-
-    Returns:
-        dict: The response from the API
-    """
-    run_async(_log_figure_async, figure)
-
 
 __all__ = [  # noqa
     "__version__",
@@ -117,8 +81,5 @@ __all__ = [  # noqa
     "test_suites",
     "vm_models",
     "unit_metrics",
-    # Framework Logging API
-    "log_figure",
-    "log_metrics",
-    "log_test_results",
+    "log_metric",
 ]

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.5.19"
+__version__ = "2.5.24"

validmind/ai/test_result_description/context.py

@@ -37,11 +37,11 @@ class Context:
         pass
 
     def load(self, input_data):
-        # this task can accept a dict or a test result object from the dev framework
+        # this task can accept a dict or a test result object from the library
         if isinstance(input_data, dict):
             return input_data
 
-        # we are likely running outside of the dev framework and need to convert
+        # we are likely running outside of the library and need to convert
         # the test result object to a dictionary
         test_result = input_data
 

validmind/api_client.py

@@ -12,7 +12,7 @@ import atexit
 import json
 import os
 from io import BytesIO
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 from urllib.parse import urlencode, urljoin
 
 import aiohttp
@@ -20,7 +20,7 @@ import requests
 from aiohttp import FormData
 
 from .client_config import client_config
-from .errors import MissingAPICredentialsError, MissingProjectIdError, raise_api_error
+from .errors import MissingAPICredentialsError, MissingModelIdError, raise_api_error
 from .logging import get_logger, init_sentry, send_single_error
 from .utils import NumpyEncoder, run_async
 from .vm_models import Figure, MetricResult, ThresholdTestResults
@@ -33,33 +33,25 @@ _api_key = os.getenv("VM_API_KEY")
 _api_secret = os.getenv("VM_API_SECRET")
 _api_host = os.getenv("VM_API_HOST")
 _model_cuid = os.getenv("VM_API_MODEL")
-_run_cuid = os.getenv("VM_RUN_CUID")
 _monitoring = False
 
-__api_session: aiohttp.ClientSession = None
+__api_session: Optional[aiohttp.ClientSession] = None
 
 
 @atexit.register
 def _close_session():
-    """Closes the async client session"""
+    """Closes the async client session at exit"""
     global __api_session
 
-    if __api_session is not None:
+    if __api_session and not __api_session.closed:
         try:
-            asyncio.run(__api_session.close())
-        except Exception:
-            pass
-
-
-def get_api_config() -> Dict[str, Optional[str]]:
-    return {
-        "VM_API_KEY": _api_key,
-        "VM_API_SECRET": _api_secret,
-        "VM_API_HOST": _api_host,
-        "VM_API_MODEL": _model_cuid,
-        "VM_RUN_CUID": _run_cuid,
-        "X-MONITORING": _monitoring,
-    }
+            loop = asyncio.get_event_loop()
+            if loop.is_running():
+                loop.create_task(__api_session.close())
+            else:
+                loop.run_until_complete(__api_session.close())
+        except Exception as e:
+            logger.exception("Error closing aiohttp session at exit: %s", e)
 
 
 def get_api_host() -> Optional[str]:
@@ -70,169 +62,36 @@ def get_api_model() -> Optional[str]:
     return _model_cuid
 
 
-def get_api_headers() -> Dict[str, str]:
+def _get_api_headers() -> Dict[str, str]:
     return {
         "X-API-KEY": _api_key,
         "X-API-SECRET": _api_secret,
-        "X-PROJECT-CUID": _model_cuid,
-        "X-MONITORING": _monitoring,
+        "X-MODEL-CUID": _model_cuid,
+        "X-MONITORING": str(_monitoring),
     }
 
 
-def init(
-    project: Optional[str] = None,
-    api_key: Optional[str] = None,
-    api_secret: Optional[str] = None,
-    api_host: Optional[str] = None,
-    model: Optional[str] = None,
-    monitoring=False,
-):
-    """
-    Initializes the API client instances and calls the /ping endpoint to ensure
-    the provided credentials are valid and we can connect to the ValidMind API.
-
-    If the API key and secret are not provided, the client will attempt to
-    retrieve them from the environment variables `VM_API_KEY` and `VM_API_SECRET`.
-
-    Args:
-        project (str, optional): The project CUID. Alias for model. Defaults to None.
-        model (str, optional): The model CUID. Defaults to None.
-        api_key (str, optional): The API key. Defaults to None.
-        api_secret (str, optional): The API secret. Defaults to None.
-        api_host (str, optional): The API host. Defaults to None.
-        monitoring (str, optional): The ongoing monitoring flag. Defaults to False.
-
-    Raises:
-        ValueError: If the API key and secret are not provided
-    """
-    global _api_key, _api_secret, _api_host, _run_cuid, _model_cuid, _monitoring
-
-    if api_key == "...":
-        # special case to detect when running a notebook with the standard init snippet
-        # will override with environment variables so we don't have to keep updating
-        # the notebook
-        api_host = api_key = api_secret = project = None
-
-    _model_cuid = project or model or os.getenv("VM_API_MODEL")
-
-    if _model_cuid is None:
-        raise MissingProjectIdError()
-
-    _api_key = api_key or os.getenv("VM_API_KEY")
-    _api_secret = api_secret or os.getenv("VM_API_SECRET")
-
-    if _api_key is None or _api_secret is None:
-        raise MissingAPICredentialsError()
-
-    _api_host = api_host or os.getenv(
-        "VM_API_HOST", "http://127.0.0.1:5000/api/v1/tracking/"
-    )
-
-    _run_cuid = os.getenv("VM_RUN_CUID", None)
-
-    _monitoring = monitoring
-
-    try:
-        __ping()
-    except Exception as e:
-        # if the api host is https, assume we're not in dev mode and send to sentry
-        if _api_host.startswith("https://"):
-            send_single_error(e)
-        raise e
-
-
 def _get_session() -> aiohttp.ClientSession:
     """Initializes the async client session"""
     global __api_session
 
-    if __api_session is None:
-        __api_session = aiohttp.ClientSession(loop=asyncio.get_event_loop())
-        __api_session.headers.update(
-            {
-                "X-API-KEY": _api_key,
-                "X-API-SECRET": _api_secret,
-                "X-PROJECT-CUID": _model_cuid,
-                "X-MONITORING": str(_monitoring),
-            }
+    if not __api_session or __api_session.closed:
+        __api_session = aiohttp.ClientSession(
+            headers=_get_api_headers(),
+            timeout=aiohttp.ClientTimeout(total=30),
         )
 
     return __api_session
 
 
-def __ping() -> Dict[str, Any]:
-    """Validates that we can connect to the ValidMind API (does not use the async session)"""
-    r = requests.get(
-        __get_url("ping", should_start_run=False),
-        headers={
-            "X-API-KEY": _api_key,
-            "X-API-SECRET": _api_secret,
-            "X-PROJECT-CUID": _model_cuid,
-            "X-MONITORING": str(_monitoring),
-        },
-    )
-    if r.status_code != 200:
-        raise_api_error(r.text)
-
-    client_info = r.json()
-
-    init_sentry(client_info.get("sentry_config", {}))
-
-    # Only show this confirmation the first time we connect to the API
-    ack_connected = False
-    if client_config.project is None:
-        ack_connected = True
-
-    # TODO: use model object when backend moves away from project
-    client_config.project = client_info["project"]
-    client_config.documentation_template = client_info.get("documentation_template", {})
-    client_config.feature_flags = client_info.get("feature_flags", {})
-    client_config.model = client_info.get("model", {})
-    client_config.document_type = client_info.get(
-        "document_type", "model_documentation"
-    )
-
-    if ack_connected:
-        if client_config.model:
-            logger.info(
-                f"🎉 Connected to ValidMind!\n"
-                f"📊 Model: {client_config.model.get('name', 'N/A')} "
-                f"(ID: {client_config.model.get('cuid', 'N/A')})\n"
-                f"📁 Document Type: {client_config.document_type}"
-            )
-        else:
-            logger.info(
-                f"Connected to ValidMind... Current Model: {client_config.project['name']}"
-                f" ({client_config.project['cuid']})"
-            )
-
-
-def reload():
-    """Reconnect to the ValidMind API and reload the project configuration"""
-
-    try:
-        __ping()
-    except Exception as e:
-        # if the api host is https, assume we're not in dev mode and send to sentry
-        if _api_host.startswith("https://"):
-            send_single_error(e)
-        raise e
-
-
-def __get_url(
+def _get_url(
     endpoint: str,
     params: Optional[Dict[str, str]] = None,
-    should_start_run: bool = True,
 ) -> str:
     global _api_host
 
     params = params or {}
 
-    if not _run_cuid and should_start_run:
-        start_run()
-
-    if should_start_run:
-        params["run_cuid"] = _run_cuid
-
     if not _api_host.endswith("/"):
         _api_host += "/"
 
@@ -245,9 +104,8 @@ def __get_url(
 async def _get(
     endpoint: str, params: Optional[Dict[str, str]] = None
 ) -> Dict[str, Any]:
-    url = __get_url(endpoint, params)
+    url = _get_url(endpoint, params)
     session = _get_session()
-    session.headers.update({"X-RUN-CUID": _run_cuid})
 
     async with session.get(url) as r:
         if r.status != 200:
@@ -262,9 +120,8 @@ async def _post(
     data: Optional[Union[dict, FormData]] = None,
     files: Optional[Dict[str, Tuple[str, BytesIO, str]]] = None,
 ) -> Dict[str, Any]:
-    url = __get_url(endpoint, params)
+    url = _get_url(endpoint, params)
     session = _get_session()
-    session.headers.update({"X-RUN-CUID": _run_cuid})
 
     if not isinstance(data, (dict)) and files is not None:
         raise ValueError("Cannot pass both non-json data and file objects to _post")
@@ -292,20 +149,99 @@ async def _post(
         return await r.json()
 
 
-async def get_metadata(content_id: str) -> Dict[str, Any]:
-    """Gets a metadata object from ValidMind API.
+def _ping() -> Dict[str, Any]:
+    """Validates that we can connect to the ValidMind API (does not use the async session)"""
+    r = requests.get(
+        url=_get_url("ping"),
+        headers=_get_api_headers(),
+    )
+    if r.status_code != 200:
+        raise_api_error(r.text)
+
+    client_info = r.json()
+
+    init_sentry(client_info.get("sentry_config", {}))
+
+    # Only show this confirmation the first time we connect to the API
+    ack_connected = not client_config.model
+
+    client_config.documentation_template = client_info.get("documentation_template", {})
+    client_config.feature_flags = client_info.get("feature_flags", {})
+    client_config.model = client_info.get("model", {})
+    client_config.document_type = client_info.get(
+        "document_type", "model_documentation"
+    )
+
+    if ack_connected:
+        logger.info(
+            f"🎉 Connected to ValidMind!\n"
+            f"📊 Model: {client_config.model.get('name', 'N/A')} "
+            f"(ID: {client_config.model.get('cuid', 'N/A')})\n"
+            f"📁 Document Type: {client_config.document_type}"
+        )
+
+
+def init(
+    project: Optional[str] = None,
+    api_key: Optional[str] = None,
+    api_secret: Optional[str] = None,
+    api_host: Optional[str] = None,
+    model: Optional[str] = None,
+    monitoring=False,
+):
+    """
+    Initializes the API client instances and calls the /ping endpoint to ensure
+    the provided credentials are valid and we can connect to the ValidMind API.
+
+    If the API key and secret are not provided, the client will attempt to
+    retrieve them from the environment variables `VM_API_KEY` and `VM_API_SECRET`.
 
     Args:
-        content_id (str): Unique content identifier for the metadata
+        project (str, optional): The project CUID. Alias for model. Defaults to None. [DEPRECATED]
+        model (str, optional): The model CUID. Defaults to None.
+        api_key (str, optional): The API key. Defaults to None.
+        api_secret (str, optional): The API secret. Defaults to None.
+        api_host (str, optional): The API host. Defaults to None.
+        monitoring (str, optional): The ongoing monitoring flag. Defaults to False.
 
     Raises:
-        Exception: If the API call fails
-
-    Returns:
-        dict: Metadata object
+        ValueError: If the API key and secret are not provided
     """
-    # TODO: add a more accurate type hint/documentation
-    return await _get(f"get_metadata/{content_id}")
+    global _api_key, _api_secret, _api_host, _model_cuid, _monitoring
+
+    if api_key == "...":
+        # special case to detect when running a notebook placeholder (...)
+        # will override with environment variables for easier local development
+        api_host = api_key = api_secret = project = None
+
+    _model_cuid = project or model or os.getenv("VM_API_MODEL")
+    if _model_cuid is None:
+        raise MissingModelIdError()
+
+    _api_key = api_key or os.getenv("VM_API_KEY")
+    _api_secret = api_secret or os.getenv("VM_API_SECRET")
+    if _api_key is None or _api_secret is None:
+        raise MissingAPICredentialsError()
+
+    _api_host = api_host or os.getenv(
+        "VM_API_HOST", "http://localhost:5000/api/v1/tracking/"
+    )
+
+    _monitoring = monitoring
+
+    reload()
+
+
+def reload():
+    """Reconnect to the ValidMind API and reload the project configuration"""
+
+    try:
+        _ping()
+    except Exception as e:
+        # if the api host is https, assume we're not in dev mode and send to sentry
+        if _api_host.startswith("https://"):
+            send_single_error(e)
+        raise e
 
 
 async def log_figure(figure: Figure) -> Dict[str, Any]:
@@ -331,19 +267,20 @@ async def log_figure(figure: Figure) -> Dict[str, Any]:
         raise e
 
 
-async def log_figures(figures: List[Figure]) -> Dict[str, Any]:
-    """Logs a list of figures
+async def get_metadata(content_id: str) -> Dict[str, Any]:
+    """Gets a metadata object from ValidMind API.
 
     Args:
-        figures (list): A list of Figure objects
+        content_id (str): Unique content identifier for the metadata
 
     Raises:
         Exception: If the API call fails
 
     Returns:
-        dict: The response from the API
+        dict: Metadata object
     """
-    return await asyncio.gather(*[log_figure(figure) for figure in figures])
+    # TODO: add a more accurate type hint/documentation
+    return await _get(f"get_metadata/{content_id}")
 
 
 async def log_metadata(
@@ -380,8 +317,8 @@ async def log_metadata(
         raise e
 
 
-async def log_metrics(
-    metrics: List[MetricResult],
+async def log_metric_result(
+    metric: MetricResult,
     inputs: List[str],
     output_template: str = None,
     section_id: str = None,
@@ -390,7 +327,7 @@ async def log_metrics(
     """Logs metrics to ValidMind API.
 
     Args:
-        metrics (list): A list of MetricResult objects
+        metric (MetricResult): A MetricResult object
         inputs (list): A list of input keys (names) that were used to run the test
         output_template (str): The optional output template for the test
         section_id (str): The section ID add a test driven block to the documentation
@@ -408,24 +345,18 @@ async def log_metrics(
     if position is not None:
         request_params["position"] = position
 
-    data = []
-
-    for metric in metrics:
-        metric_data = {
-            **metric.serialize(),
-            "inputs": inputs,
-        }
-
-        if output_template:
-            metric_data["output_template"] = output_template
-
-        data.append(metric_data)
+    metric_data = {
+        **metric.serialize(),
+        "inputs": inputs,
+    }
+    if output_template:
+        metric_data["output_template"] = output_template
 
     try:
         return await _post(
             "log_metrics",
             params=request_params,
-            data=json.dumps(data, cls=NumpyEncoder, allow_nan=False),
+            data=json.dumps([metric_data], cls=NumpyEncoder, allow_nan=False),
         )
     except Exception as e:
         logger.error("Error logging metrics to ValidMind API")
@@ -479,35 +410,6 @@ async def log_test_result(
         raise e
 
 
-def log_test_results(
-    results: List[ThresholdTestResults], inputs
-) -> List[Callable[..., Dict[str, Any]]]:
-    """Logs test results information
-
-    This method will be called automatically be any function
-    running tests but can also be called directly if the user wants to run tests on their own.
-
-    Args:
-        results (list): A list of ThresholdTestResults objects
-        inputs (list): A list of input IDs that were used to run the test
-
-    Raises:
-        Exception: If the API call fails
-
-    Returns:
-        list: list of responses from the API
-    """
-    try:
-        responses = []  # TODO: use asyncio.gather
-        for result in results:
-            responses.append(run_async(log_test_result, result, inputs))
-    except Exception as e:
-        logger.error("Error logging test results to ValidMind API")
-        raise e
-
-    return responses
-
-
 def log_input(input_id: str, type: str, metadata: Dict[str, Any]) -> Dict[str, Any]:
     """Logs input information - internal use for now (don't expose via public API)
 
@@ -547,7 +449,7 @@ async def alog_metric(
     inputs: Optional[List[str]] = None,
     params: Optional[Dict[str, Any]] = None,
     recorded_at: Optional[str] = None,
-) -> None:
+):
     """See log_metric for details"""
     if not key or not isinstance(key, str):
         raise ValueError("`key` must be a non-empty string")
@@ -581,7 +483,7 @@ def log_metric(
     inputs: Optional[List[str]] = None,
     params: Optional[Dict[str, Any]] = None,
     recorded_at: Optional[str] = None,
-) -> None:
+):
     """Logs a unit metric
 
     Unit metrics are key-value pairs where the key is the metric name and the value is
@@ -601,48 +503,11 @@ def log_metric(
     run_async(alog_metric, key, value, inputs, params, recorded_at)
 
 
-def start_run() -> str:
-    """Starts a new test run
-
-    This function will take care of updating the api client with the new run CUID
-    and will be called automatically when logging starts if no run CUID is set.
-
-    Raises:
-        Exception: If the API call fails
-
-    Returns:
-        str: The test run CUID
-    """
-    global _run_cuid
-
-    r = requests.post(
-        __get_url("start_run", should_start_run=False),
-        headers={
-            "X-API-KEY": _api_key,
-            "X-API-SECRET": _api_secret,
-            "X-PROJECT-CUID": _model_cuid,
-        },
-    )
-
-    if r.status_code != 200:
-        logger.error("Could not start data logging run with ValidMind API")
-        raise_api_error(r.text)
-
-    test_run = r.json()
-    _run_cuid = test_run["cuid"]
-
-    return test_run["cuid"]
-
-
-def get_ai_key() -> str:
+def get_ai_key() -> Dict[str, Any]:
     """Calls the api to get an api key for our LLM proxy"""
     r = requests.get(
-        __get_url("ai/key", should_start_run=False),
-        headers={
-            "X-API-KEY": _api_key,
-            "X-API-SECRET": _api_secret,
-            "X-PROJECT-CUID": _model_cuid,
-        },
+        url=_get_url("ai/key"),
+        headers=_get_api_headers(),
     )
 
     if r.status_code != 200:

validmind/client_config.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-Central class to track configuration of the developer framework
+Central class to track configuration of the library
 client against the ValidMind API
 """
 
@@ -17,7 +17,6 @@ class ClientConfig:
     when initializing the API client.
     """
 
-    project: object
     model: object
     feature_flags: dict
     document_type: str
@@ -42,7 +41,6 @@ class ClientConfig:
 
 
 client_config = ClientConfig(
-    project=None,
     model=None,
     feature_flags={},
     document_type="model_documentation",

validmind/datasets/__init__.py

@@ -3,5 +3,5 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-Example datasets that can be used with the developer framework.
+Example datasets that can be used with the library.
 """