validmind 2.5.25__tar.gz → 2.6.7__tar.gz

validmind-2.6.7/PKG-INFO

@@ -0,0 +1,137 @@
+Metadata-Version: 2.1
+Name: validmind
+Version: 2.6.7
+Summary: ValidMind Library
+License: Commercial License
+Author: Andres Rodriguez
+Author-email: andres@validmind.ai
+Requires-Python: >=3.8.1,<3.12
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Provides-Extra: all
+Provides-Extra: huggingface
+Provides-Extra: llm
+Provides-Extra: pytorch
+Provides-Extra: r-support
+Requires-Dist: aiohttp[speedups]
+Requires-Dist: arch
+Requires-Dist: bert-score (>=0.3.13)
+Requires-Dist: catboost
+Requires-Dist: datasets (>=2.10.0,<3.0.0)
+Requires-Dist: evaluate
+Requires-Dist: ipywidgets
+Requires-Dist: kaleido (>=0.2.1,!=0.2.1.post1)
+Requires-Dist: langchain-openai (>=0.1.8) ; extra == "all" or extra == "llm"
+Requires-Dist: langdetect
+Requires-Dist: latex2mathml (>=3.77.0)
+Requires-Dist: llvmlite ; python_version >= "3.8" and python_full_version <= "3.11.0"
+Requires-Dist: matplotlib
+Requires-Dist: mistune (>=3.0.2,<4.0.0)
+Requires-Dist: nest-asyncio (>=1.6.0,<2.0.0)
+Requires-Dist: nltk (>=3.8.1,<4.0.0)
+Requires-Dist: numba (<0.59.0)
+Requires-Dist: numpy
+Requires-Dist: openai (>=1)
+Requires-Dist: pandas (>=1.1,<=2.0.3)
+Requires-Dist: plotly
+Requires-Dist: plotly-express
+Requires-Dist: polars
+Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"
+Requires-Dist: python-dotenv
+Requires-Dist: ragas (>=0.2.3) ; extra == "all" or extra == "llm"
+Requires-Dist: rouge (>=1)
+Requires-Dist: rpy2 (>=3.5.10,<4.0.0) ; extra == "all" or extra == "r-support"
+Requires-Dist: scikit-learn
+Requires-Dist: scipy
+Requires-Dist: scorecardpy (>=0.1.9.6,<0.2.0.0)
+Requires-Dist: seaborn
+Requires-Dist: sentencepiece (>=0.2.0,<0.3.0) ; extra == "all" or extra == "huggingface" or extra == "llm"
+Requires-Dist: sentry-sdk (>=1.24.0,<2.0.0)
+Requires-Dist: shap (==0.44.1)
+Requires-Dist: statsmodels
+Requires-Dist: tabulate (>=0.8.9,<0.9.0)
+Requires-Dist: textblob (>=0.18.0.post0,<0.19.0)
+Requires-Dist: torch (>=1.10.0) ; extra == "all" or extra == "llm" or extra == "pytorch"
+Requires-Dist: tqdm
+Requires-Dist: transformers (>=4.32.0,<5.0.0) ; extra == "all" or extra == "huggingface" or extra == "llm"
+Requires-Dist: xgboost (>=1.5.2,<3)
+Requires-Dist: ydata-profiling
+Requires-Dist: yfinance (>=0.2.48,<0.3.0)
+Description-Content-Type: text/markdown
+
+# ValidMind Library
+
+The ValidMind Library is a suite of developer tools and methods designed to automate the documentation and validation of your models.
+
+Designed to be model agnostic, the ValidMind Library provides all the standard functionality without requiring you to rewrite any functions as long as your model is built in Python.
+
+With a rich array of documentation tools and test suites, from documenting descriptions of your datasets to testing your models for weak spots and overfit areas, the ValidMind Library helps you automate model documentation by feeding the ValidMind Platform with documentation artifacts and test results.
+
+## What is ValidMind?
+
+ValidMind helps developers, data scientists and risk and compliance stakeholders identify potential risks in their AI and large language models, and generate robust, high-quality model documentation that meets regulatory requirements.
+
+[The ValidMind AI risk platform](https://docs.validmind.ai/about/overview.html) consists of two intertwined product offerings:
+
+- **The ValidMind Library** — Designed to be incorporated into your existing model development environment, you use the ValidMind Library to run tests and log documentation to the ValidMind Platform. Driven by the power of open-source, the ValidMind Library welcomes contributions to our code and developer samples: [`validmind-library` @ GitHub](https://github.com/validmind/validmind-library)
+- **The ValidMind Platform** — A cloud-hosted user interface allowing you to comprehensively track your model inventory throughout the entire model lifecycle according to the unique requirements of your organization. You use the ValidMind Platform to oversee your model risk management process via the customizable model inventory.
+
+### What do I need to get started with ValidMind?
+
+> **All you need to get started with ValidMind is an account with us.**
+>
+> Signing up is FREE — **[Register with ValidMind](https://docs.validmind.ai/guide/configuration/register-with-validmind.html)**
+
+That's right — you can run tests and log documentation even if you don't have a model available, so go ahead and [**Get started with the ValidMind Library**](https://docs.validmind.ai/developer/get-started-validmind-library.html)!
+
+### How do I do more with the ValidMind Library?
+
+**[Explore our code samples!](https://docs.validmind.ai/developer/samples-jupyter-notebooks.html)**
+
+Our selection of Jupyter Notebooks showcase the capabilities and features of the ValidMind Library, while also providing you with useful examples that you can build on and adapt for your own use cases.
+
+## Installation
+
+To install the ValidMind Library and all optional dependencies, run:
+
+```bash
+pip install validmind[all]
+```
+
+To install the ValidMind Library without optional dependencies (core functionality only), run:
+
+```bash
+pip install validmind
+```
+
+### Extra dependencies
+
+The ValidMind Library has optional dependencies that can be installed separately to support additional model types and tests.
+
+- **LLM Support**: To be able to run tests for Large Language Models (LLMs), install the `llm` extra:
+
+    ```bash
+    pip install validmind[llm]
+    ```
+
+- **PyTorch Models**: To use pytorch models with the ValidMind Library, install the `torch` extra:
+
+    ```bash
+    pip install validmind[torch]
+    ```
+
+- **Hugging Face Transformers**: To use Hugging Face Transformers models with the ValidMind Library, install the `transformers` extra:
+
+    ```bash
+    pip install validmind[transformers]
+    ```
+
+- **R Models**: To use R models with the ValidMind Library, install the `r` extra:
+
+    ```bash
+    pip install validmind[r-support]
+    ```
+

validmind-2.6.7/README.pypi.md

@@ -0,0 +1,72 @@
+# ValidMind Library
+
+The ValidMind Library is a suite of developer tools and methods designed to automate the documentation and validation of your models.
+
+Designed to be model agnostic, the ValidMind Library provides all the standard functionality without requiring you to rewrite any functions as long as your model is built in Python.
+
+With a rich array of documentation tools and test suites, from documenting descriptions of your datasets to testing your models for weak spots and overfit areas, the ValidMind Library helps you automate model documentation by feeding the ValidMind Platform with documentation artifacts and test results.
+
+## What is ValidMind?
+
+ValidMind helps developers, data scientists and risk and compliance stakeholders identify potential risks in their AI and large language models, and generate robust, high-quality model documentation that meets regulatory requirements.
+
+[The ValidMind AI risk platform](https://docs.validmind.ai/about/overview.html) consists of two intertwined product offerings:
+
+- **The ValidMind Library** — Designed to be incorporated into your existing model development environment, you use the ValidMind Library to run tests and log documentation to the ValidMind Platform. Driven by the power of open-source, the ValidMind Library welcomes contributions to our code and developer samples: [`validmind-library` @ GitHub](https://github.com/validmind/validmind-library)
+- **The ValidMind Platform** — A cloud-hosted user interface allowing you to comprehensively track your model inventory throughout the entire model lifecycle according to the unique requirements of your organization. You use the ValidMind Platform to oversee your model risk management process via the customizable model inventory.
+
+### What do I need to get started with ValidMind?
+
+> **All you need to get started with ValidMind is an account with us.**
+>
+> Signing up is FREE — **[Register with ValidMind](https://docs.validmind.ai/guide/configuration/register-with-validmind.html)**
+
+That's right — you can run tests and log documentation even if you don't have a model available, so go ahead and [**Get started with the ValidMind Library**](https://docs.validmind.ai/developer/get-started-validmind-library.html)!
+
+### How do I do more with the ValidMind Library?
+
+**[Explore our code samples!](https://docs.validmind.ai/developer/samples-jupyter-notebooks.html)**
+
+Our selection of Jupyter Notebooks showcase the capabilities and features of the ValidMind Library, while also providing you with useful examples that you can build on and adapt for your own use cases.
+
+## Installation
+
+To install the ValidMind Library and all optional dependencies, run:
+
+```bash
+pip install validmind[all]
+```
+
+To install the ValidMind Library without optional dependencies (core functionality only), run:
+
+```bash
+pip install validmind
+```
+
+### Extra dependencies
+
+The ValidMind Library has optional dependencies that can be installed separately to support additional model types and tests.
+
+- **LLM Support**: To be able to run tests for Large Language Models (LLMs), install the `llm` extra:
+
+    ```bash
+    pip install validmind[llm]
+    ```
+
+- **PyTorch Models**: To use pytorch models with the ValidMind Library, install the `torch` extra:
+
+    ```bash
+    pip install validmind[torch]
+    ```
+
+- **Hugging Face Transformers**: To use Hugging Face Transformers models with the ValidMind Library, install the `transformers` extra:
+
+    ```bash
+    pip install validmind[transformers]
+    ```
+
+- **R Models**: To use R models with the ValidMind Library, install the `r` extra:
+
+    ```bash
+    pip install validmind[r-support]
+    ```

{validmind-2.5.25 → validmind-2.6.7}/pyproject.toml

@@ -10,7 +10,7 @@ description = "ValidMind Library"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.5.25"
+version = "2.6.7"
 
 [tool.poetry.dependencies]
 aiohttp = {extras = ["speedups"], version = "*"}
@@ -29,7 +29,7 @@ matplotlib = "*"
 mistune = "^3.0.2"
 nest-asyncio = "^1.6.0"
 nltk = "^3.8.1"
-numba = "<0.59.0" # TODO: https://github.com/validmind/developer-framework/pull/28
+numba = "<0.59.0" # TODO: https://github.com/validmind/validmind-library/pull/28
 numpy = "*"
 openai = ">=1"
 pandas = ">=1.1,<=2.0.3"
@@ -39,7 +39,7 @@ polars = "*"
 pycocoevalcap = {version = "^1.2", optional = true}
 python = ">=3.8.1,<3.12"
 python-dotenv = "*"
-ragas = {version = ">=0.1.19,<0.1.22", optional = true}
+ragas = {version = ">=0.2.3", optional = true}
 rouge = ">=1"
 rpy2 = {version = "^3.5.10", optional = true}
 scikit-learn = "*"
@@ -57,6 +57,7 @@ tqdm = "*"
 transformers = {version = "^4.32.0", optional = true}
 xgboost = ">=1.5.2,<3"
 ydata-profiling = "*"
+yfinance = "^0.2.48"
 
 [tool.poetry.group.dev.dependencies]
 black = "^22.1.0"

{validmind-2.5.25 → validmind-2.6.7}/validmind/__init__.py

@@ -3,25 +3,19 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 """
-ValidMind’s Python Library is a library of developer tools and methods designed to automate
-the documentation and validation of your models.
+The ValidMind Library is a suite of developer tools and methods designed to automate the documentation and validation of your models.
 
-The Library is designed to be model agnostic. If your model is built in Python, ValidMind's
-Python library will provide all the standard functionality without requiring your developers to rewrite any functions.
+Designed to be model agnostic, the ValidMind Library provides all the standard functionality without requiring you to rewrite any functions as long as your model is built in Python.
 
-The Library provides a rich suite of documentation tools and test suites, from documenting
-descriptions of your dataset to testing your models for weak spots and overfit areas. The Library
-helps you automate the generation of model documentation by feeding the ValidMind platform with documentation
-artifacts and test results to the ValidMind platform.
+With a rich array of documentation tools and test suites, from documenting descriptions of your datasets to testing your models for weak spots and overfit areas, the ValidMind Library helps you automate model documentation by feeding the ValidMind Platform with documentation artifacts and test results.
 
-To install the client library:
+To install the ValidMind Library:
 
 ```bash
 pip install validmind
 ```
 
-To initialize the client library, paste the code snippet with the client integration details directly into your
-development source code, replacing this example with your own:
+To initialize the ValidMind Library, paste the code snippet with the model identifier credentials directly into your development source code, replacing this example with your own:
 
 ```python
 import validmind as vm
@@ -34,9 +28,7 @@ vm.init(
 )
 ```
 
-After you have pasted the code snippet into your development source code and executed the code, the Python client
-library will register with ValidMind. You can now use the Library to document and test your models,
-and to upload to the ValidMind Platform.
+After you have pasted the code snippet into your development source code and executed the code, the Python Library API will register with ValidMind. You can now use the ValidMind Library to document and test your models, and to upload to the ValidMind Platform.
 """
 import warnings
 
@@ -57,11 +49,11 @@ from .client import (  # noqa: E402
     run_documentation_tests,
     run_test_suite,
 )
-from .tests.decorator import metric, tags, tasks, test
+from .tests.decorator import tags, tasks, test
 
 __all__ = [  # noqa
     "__version__",
-    # Framework High Level API
+    # Python Library API
     "datasets",
     "errors",
     "get_test_suite",
@@ -69,7 +61,6 @@ __all__ = [  # noqa
     "init_dataset",
     "init_model",
     "init_r_model",
-    "metric",
     "preview_template",
     "reload",
     "run_documentation_tests",

validmind-2.6.7/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.6.7"

{validmind-2.5.25 → validmind-2.6.7}/validmind/ai/test_descriptions.py

@@ -2,17 +2,20 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import json
 import os
 import re
 from concurrent.futures import ThreadPoolExecutor
-from typing import Union
+from typing import List, Optional, Union
 
 from jinja2 import Template
 
-from validmind.utils import md_to_html
-
 from ..client_config import client_config
 from ..logging import get_logger
+from ..utils import NumpyEncoder, md_to_html, test_id_to_name
+from ..vm_models.figure import Figure
+from ..vm_models.result import ResultTable
+from .utils import DescriptionFuture, get_client_and_model
 
 __executor = ThreadPoolExecutor()
 __prompt = None
@@ -20,10 +23,6 @@ __prompt = None
 logger = get_logger(__name__)
 
 
-AI_REVISION_NAME = "Generated by ValidMind AI"
-DEFAULT_REVISION_NAME = "Default Description"
-
-
 def _load_prompt():
     global __prompt
 
@@ -55,7 +54,6 @@ def prompt_to_message(role, prompt):
         if start > last_index:
             content.append({"type": "text", "text": prompt[last_index:start]})
 
-        # Image
         content.append({"type": "image_url", "image_url": {"url": match.group(1)}})
 
         last_index = end
@@ -67,79 +65,66 @@ def prompt_to_message(role, prompt):
     return {"role": role, "content": content}
 
 
-class DescriptionFuture:
-    """This will be immediately returned from generate_description so that
-    the tests can continue to be run in parallel while the description is
-    retrieved asynchronously.
-
-    The value will be retrieved later and if its not ready yet, it should
-    block until it is.
-    """
-
-    def __init__(self, future):
-        self._future = future
-
-    def get_description(self):
-        if isinstance(self._future, str):
-            description = self._future
-        else:
-            # This will block until the future is completed
-            description = self._future.result()
-
-        return md_to_html(description, mathml=True)
-
-
 def generate_description(
     test_id: str,
     test_description: str,
-    test_summary: str,
+    tables: List[ResultTable] = None,
     metric: Union[float, int] = None,
-    figures: list = None,
+    figures: List[Figure] = None,
+    title: Optional[str] = None,
 ):
     """Generate the description for the test results"""
-    if not test_summary and not figures and not metric:
+    if not tables and not figures and not metric:
         raise ValueError(
-            "No summary, unit metric or figures provided - cannot generate description"
+            "No tables, unit metric or figures provided - cannot generate description"
         )
 
-    # TODO: fix circular import
-    from validmind.ai.utils import get_client_and_model
+    # # TODO: fix circular import
+    # from validmind.ai.utils import get_client_and_model
 
     client, model = get_client_and_model()
 
     # get last part of test id
-    test_name = test_id.split(".")[-1]
-    # truncate the test description to save time
-    test_description = (
-        f"{test_description[:500]}..."
-        if len(test_description) > 500
-        else test_description
-    )
-
-    if metric:
-        metric_summary = f"**Metric Value**: {metric}"
-        if test_summary:
-            test_summary = metric_summary + "\n" + test_summary
-        else:
-            test_summary = metric_summary
+    test_name = title or test_id.split(".")[-1]
+
+    # TODO: fully support metrics
+    if metric is not None:
+        tables = [] if not tables else tables
+        tables.append(
+            ResultTable(
+                data=[
+                    {"Metric": test_id_to_name(test_id), "Value": metric},
+                ],
+            )
+        )
 
-    figures = [] if test_summary else figures
+    if tables:
+        summary = "\n---\n".join(
+            [
+                json.dumps(table.serialize(), cls=NumpyEncoder, separators=(",", ":"))
+                for table in tables
+            ]
+        )
+    else:
+        summary = None
 
     input_data = {
         "test_name": test_name,
         "test_description": test_description,
-        "summary": test_summary,
-        "figures": [figure._get_b64_url() for figure in figures],
+        "title": title,
+        "summary": summary,
+        "figures": [figure._get_b64_url() for figure in ([] if tables else figures)],
     }
     system, user = _load_prompt()
 
+    messages = [
+        prompt_to_message("system", system.render(input_data)),
+        prompt_to_message("user", user.render(input_data)),
+    ]
     response = client.chat.completions.create(
         model=model,
         temperature=0.0,
-        messages=[
-            prompt_to_message("system", system.render(input_data)),
-            prompt_to_message("user", user.render(input_data)),
-        ],
+        messages=messages,
     )
 
     return response.choices[0].message.content
@@ -148,18 +133,20 @@ def generate_description(
 def background_generate_description(
     test_id: str,
     test_description: str,
-    test_summary: str,
-    figures: list = None,
+    tables: List[ResultTable] = None,
+    figures: List[Figure] = None,
     metric: Union[int, float] = None,
+    title: Optional[str] = None,
 ):
     def wrapped():
         try:
             return generate_description(
                 test_id=test_id,
                 test_description=test_description,
-                test_summary=test_summary,
+                tables=tables,
                 figures=figures,
                 metric=metric,
+                title=title,
             )
         except Exception as e:
             logger.error(f"Failed to generate description: {e}")
@@ -169,14 +156,14 @@ def background_generate_description(
     return DescriptionFuture(__executor.submit(wrapped))
 
 
-def get_description_metadata(
-    test_id,
-    default_description,
-    summary=None,
-    figures=None,
-    metric=None,
-    prefix="metric_description",
-    should_generate=True,
+def get_result_description(
+    test_id: str,
+    test_description: str,
+    tables: List[ResultTable] = None,
+    figures: List[Figure] = None,
+    metric: Union[int, float] = None,
+    should_generate: bool = True,
+    title: Optional[str] = None,
 ):
     """Get Metadata Dictionary for a Test or Metric Result
 
@@ -184,24 +171,23 @@ def get_description_metadata(
     description and returns a metadata object that can be logged with the test results.
 
     By default, the description is generated by an LLM that will interpret the test
-    results and provide a human-readable description. If the summary or figures are
+    results and provide a human-readable description. If the tables or figures are
     not provided, or the `VALIDMIND_LLM_DESCRIPTIONS_ENABLED` environment variable is
     set to `0` or `false` or no LLM has been configured, the default description will
     be used as the test result description.
 
-    Note: Either the summary or figures must be provided to generate the description.
+    Note: Either the tables or figures must be provided to generate the description.
 
     Args:
         test_id (str): The test ID
-        default_description (str): The default description for the test
-        summary (Any): The test summary or results to interpret
+        test_description (str): The default description for the test
+        tables (Any): The test tables or results to interpret
         figures (List[Figure]): The figures to attach to the test suite result
         metric (Union[int, float]): Unit metrics attached to the test result
-        prefix (str): The prefix to use for the content ID (Default: "metric_description")
         should_generate (bool): Whether to generate the description or not (Default: True)
 
     Returns:
-        dict: The metadata object to be logged with the test results
+        str: The description to be logged with the test results
     """
     # Check the feature flag first, then the environment variable
     llm_descriptions_enabled = (
@@ -214,27 +200,22 @@ def get_description_metadata(
 
     if (
         should_generate
-        and (summary or figures)
+        and (tables or figures)
         and llm_descriptions_enabled
         and is_configured()
     ):
-        revision_name = AI_REVISION_NAME
-
         # get description future and set it as the description in the metadata
         # this will lazily retrieved so it can run in the background in parallel
         description = background_generate_description(
             test_id=test_id,
-            test_description=default_description,
-            test_summary=summary,
+            test_description=test_description,
+            tables=tables,
             figures=figures,
             metric=metric,
+            title=title,
         )
 
     else:
-        revision_name = DEFAULT_REVISION_NAME
-        description = md_to_html(default_description, mathml=True)
+        description = md_to_html(test_description, mathml=True)
 
-    return {
-        "content_id": f"{prefix}:{test_id}::{revision_name}",
-        "text": description,
-    }
+    return description

{validmind-2.5.25 → validmind-2.6.7}/validmind/ai/test_result_description/context.py

@@ -37,11 +37,11 @@ class Context:
         pass
 
     def load(self, input_data):
-        # this task can accept a dict or a test result object from the library
+        # this task can accept a dict or a test result object from the ValidMind Library
         if isinstance(input_data, dict):
             return input_data
 
-        # we are likely running outside of the library and need to convert
+        # we are likely running outside of the ValidMind Library and need to convert
         # the test result object to a dictionary
         test_result = input_data
 

{validmind-2.5.25 → validmind-2.6.7}/validmind/ai/utils.py

@@ -7,8 +7,8 @@ from urllib.parse import urljoin
 
 from openai import AzureOpenAI, Client, OpenAI
 
-from ..api_client import get_ai_key, get_api_host
 from ..logging import get_logger
+from ..utils import md_to_html
 
 logger = get_logger(__name__)
 
@@ -19,6 +19,28 @@ __model = None
 __ack = None
 
 
+class DescriptionFuture:
+    """This will be immediately returned from generate_description so that
+    the tests can continue to be run in parallel while the description is
+    retrieved asynchronously.
+
+    The value will be retrieved later and if its not ready yet, it should
+    block until it is.
+    """
+
+    def __init__(self, future):
+        self._future = future
+
+    def get_description(self):
+        if isinstance(self._future, str):
+            description = self._future
+        else:
+            # This will block until the future is completed
+            description = self._future.result()
+
+        return md_to_html(description, mathml=True)
+
+
 def get_client_and_model():
     """Get model and client to use for generating interpretations
 
@@ -58,6 +80,9 @@ def get_client_and_model():
 
     else:
         try:
+            # TODO: fix circular import
+            from ..api_client import get_ai_key, get_api_host
+
             response = get_ai_key()
             __client = Client(
                 base_url=(