validmind 2.5.8__tar.gz → 2.5.18__tar.gz

{validmind-2.5.8 → validmind-2.5.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.5.8
+Version: 2.5.18
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez
@@ -20,6 +20,7 @@ Requires-Dist: aiohttp[speedups]
 Requires-Dist: arch
 Requires-Dist: bert-score (>=0.3.13)
 Requires-Dist: catboost
+Requires-Dist: datasets (>=2.10.0,<3.0.0)
 Requires-Dist: evaluate
 Requires-Dist: ipywidgets
 Requires-Dist: kaleido (>=0.2.1,!=0.2.1.post1)
@@ -34,13 +35,13 @@ Requires-Dist: nltk (>=3.8.1,<4.0.0)
 Requires-Dist: numba (<0.59.0)
 Requires-Dist: numpy
 Requires-Dist: openai (>=1)
-Requires-Dist: pandas (>=1.1,<2)
+Requires-Dist: pandas (>=1.1,<=2.0.3)
 Requires-Dist: plotly
 Requires-Dist: plotly-express
 Requires-Dist: polars
 Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"
 Requires-Dist: python-dotenv
-Requires-Dist: ragas (>=0.1.7) ; extra == "all" or extra == "llm"
+Requires-Dist: ragas (>=0.1.19) ; extra == "all" or extra == "llm"
 Requires-Dist: rouge (>=1)
 Requires-Dist: rpy2 (>=3.5.10,<4.0.0) ; extra == "all" or extra == "r-support"
 Requires-Dist: scikit-learn

{validmind-2.5.8 → validmind-2.5.18}/pyproject.toml

@@ -10,15 +10,15 @@ description = "ValidMind Developer Framework"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.5.8"
+version = "2.5.18"
 
 [tool.poetry.dependencies]
-python = ">=3.8.1,<3.12"
 aiohttp = {extras = ["speedups"], version = "*"}
 arch = "*"
 bert-score = ">=0.3.13"
 catboost = "*"
 evaluate = "*"
+datasets = "^2.10.0"
 ipywidgets = "*"
 kaleido = ">=0.2.1,!=0.2.1.post1"
 langchain-openai = {version = ">=0.1.8", optional = true}
@@ -32,13 +32,14 @@ nltk = "^3.8.1"
 numba = "<0.59.0" # TODO: https://github.com/validmind/developer-framework/pull/28
 numpy = "*"
 openai = ">=1"
-pandas = ">=1.1,<2" # TODO: small issues blocking 2.0 but it would speed things up to use arrow backend
+pandas = ">=1.1,<=2.0.3"
 plotly = "*"
 plotly-express = "*"
 polars = "*"
 pycocoevalcap = {version = "^1.2", optional = true}
+python = ">=3.8.1,<3.12"
 python-dotenv = "*"
-ragas = {version = ">=0.1.7", optional = true}
+ragas = {version = ">=0.1.19", optional = true}
 rouge = ">=1"
 rpy2 = {version = "^3.5.10", optional = true}
 scikit-learn = "*"

validmind-2.5.18/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.5.18"

validmind-2.5.18/validmind/ai/test_descriptions.py

@@ -0,0 +1,240 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import os
+import re
+from concurrent.futures import ThreadPoolExecutor
+from typing import Union
+
+from jinja2 import Template
+
+from validmind.utils import md_to_html
+
+from ..client_config import client_config
+from ..logging import get_logger
+
+__executor = ThreadPoolExecutor()
+__prompt = None
+
+logger = get_logger(__name__)
+
+
+AI_REVISION_NAME = "Generated by ValidMind AI"
+DEFAULT_REVISION_NAME = "Default Description"
+
+
+def _load_prompt():
+    global __prompt
+
+    if not __prompt:
+        folder_path = os.path.join(os.path.dirname(__file__), "test_result_description")
+        with open(os.path.join(folder_path, "system.jinja"), "r") as f:
+            system_prompt = f.read()
+        with open(os.path.join(folder_path, "user.jinja"), "r") as f:
+            user_prompt = f.read()
+
+        __prompt = (Template(system_prompt), Template(user_prompt))
+
+    return __prompt
+
+
+def prompt_to_message(role, prompt):
+    if "[[IMAGE:" not in prompt:
+        return {"role": role, "content": prompt}
+
+    content = []
+
+    # Regex pattern to find [[IMAGE:<b64-data>]] markers
+    pattern = re.compile(r"\[\[IMAGE:(.*?)\]\]", re.DOTALL)
+
+    last_index = 0
+    for match in pattern.finditer(prompt):
+        # Text before the image marker
+        start, end = match.span()
+        if start > last_index:
+            content.append({"type": "text", "text": prompt[last_index:start]})
+
+        # Image
+        content.append({"type": "image_url", "image_url": {"url": match.group(1)}})
+
+        last_index = end
+
+    # Text after the last image
+    if last_index < len(prompt):
+        content.append({"type": "text", "text": prompt[last_index:]})
+
+    return {"role": role, "content": content}
+
+
+class DescriptionFuture:
+    """This will be immediately returned from generate_description so that
+    the tests can continue to be run in parallel while the description is
+    retrieved asynchronously.
+
+    The value will be retrieved later and if its not ready yet, it should
+    block until it is.
+    """
+
+    def __init__(self, future):
+        self._future = future
+
+    def get_description(self):
+        if isinstance(self._future, str):
+            description = self._future
+        else:
+            # This will block until the future is completed
+            description = self._future.result()
+
+        return md_to_html(description, mathml=True)
+
+
+def generate_description(
+    test_id: str,
+    test_description: str,
+    test_summary: str,
+    metric: Union[float, int] = None,
+    figures: list = None,
+):
+    """Generate the description for the test results"""
+    if not test_summary and not figures and not metric:
+        raise ValueError(
+            "No summary, unit metric or figures provided - cannot generate description"
+        )
+
+    # TODO: fix circular import
+    from validmind.ai.utils import get_client_and_model
+
+    client, model = get_client_and_model()
+
+    # get last part of test id
+    test_name = test_id.split(".")[-1]
+    # truncate the test description to save time
+    test_description = (
+        f"{test_description[:500]}..."
+        if len(test_description) > 500
+        else test_description
+    )
+
+    if metric:
+        metric_summary = f"**Metric Value**: {metric}"
+        if test_summary:
+            test_summary = metric_summary + "\n" + test_summary
+        else:
+            test_summary = metric_summary
+
+    figures = [] if test_summary else figures
+
+    input_data = {
+        "test_name": test_name,
+        "test_description": test_description,
+        "summary": test_summary,
+        "figures": [figure._get_b64_url() for figure in figures],
+    }
+    system, user = _load_prompt()
+
+    response = client.chat.completions.create(
+        model=model,
+        temperature=0.0,
+        messages=[
+            prompt_to_message("system", system.render(input_data)),
+            prompt_to_message("user", user.render(input_data)),
+        ],
+    )
+
+    return response.choices[0].message.content
+
+
+def background_generate_description(
+    test_id: str,
+    test_description: str,
+    test_summary: str,
+    figures: list = None,
+    metric: Union[int, float] = None,
+):
+    def wrapped():
+        try:
+            return generate_description(
+                test_id=test_id,
+                test_description=test_description,
+                test_summary=test_summary,
+                figures=figures,
+                metric=metric,
+            )
+        except Exception as e:
+            logger.error(f"Failed to generate description: {e}")
+
+            return test_description
+
+    return DescriptionFuture(__executor.submit(wrapped))
+
+
+def get_description_metadata(
+    test_id,
+    default_description,
+    summary=None,
+    figures=None,
+    metric=None,
+    prefix="metric_description",
+    should_generate=True,
+):
+    """Get Metadata Dictionary for a Test or Metric Result
+
+    Generates an LLM interpretation of the test results or uses the default
+    description and returns a metadata object that can be logged with the test results.
+
+    By default, the description is generated by an LLM that will interpret the test
+    results and provide a human-readable description. If the summary or figures are
+    not provided, or the `VALIDMIND_LLM_DESCRIPTIONS_ENABLED` environment variable is
+    set to `0` or `false` or no LLM has been configured, the default description will
+    be used as the test result description.
+
+    Note: Either the summary or figures must be provided to generate the description.
+
+    Args:
+        test_id (str): The test ID
+        default_description (str): The default description for the test
+        summary (Any): The test summary or results to interpret
+        figures (List[Figure]): The figures to attach to the test suite result
+        metric (Union[int, float]): Unit metrics attached to the test result
+        prefix (str): The prefix to use for the content ID (Default: "metric_description")
+        should_generate (bool): Whether to generate the description or not (Default: True)
+
+    Returns:
+        dict: The metadata object to be logged with the test results
+    """
+    # Check the feature flag first, then the environment variable
+    llm_descriptions_enabled = (
+        client_config.can_generate_llm_test_descriptions()
+        and os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1") not in ["0", "false"]
+    )
+
+    # TODO: fix circular import
+    from validmind.ai.utils import is_configured
+
+    if (
+        should_generate
+        and (summary or figures)
+        and llm_descriptions_enabled
+        and is_configured()
+    ):
+        revision_name = AI_REVISION_NAME
+
+        # get description future and set it as the description in the metadata
+        # this will lazily retrieved so it can run in the background in parallel
+        description = background_generate_description(
+            test_id=test_id,
+            test_description=default_description,
+            test_summary=summary,
+            figures=figures,
+            metric=metric,
+        )
+
+    else:
+        revision_name = DEFAULT_REVISION_NAME
+        description = md_to_html(default_description, mathml=True)
+
+    return {
+        "content_id": f"{prefix}:{test_id}::{revision_name}",
+        "text": description,
+    }

validmind-2.5.18/validmind/ai/test_result_description/config.yaml

@@ -0,0 +1,29 @@
+id: test_result_description
+name: Test Result Description
+description: Generate a description for a test result
+version: 0.1.0
+model: gpt-4o
+temperature: 0.0
+output_type: markdown
+prompts:
+  system:
+    role: system
+    path: system.jinja
+  user:
+    role: user
+    path: user.jinja
+inputs:
+  test_name:
+    description: The name of the test that produced the result (usually the last part of the test ID)
+    type: string
+  test_description:
+    description: The description (docstring) of the test that was run
+    type: string
+  summary:
+    description: The json result summary (i.e. the table(s) returned by the test)
+    type: list
+    optional: true
+  figures:
+    description: A list of base64 encoded images of the figures returned by the test
+    type: list
+    optional: true

validmind-2.5.18/validmind/ai/test_result_description/context.py

@@ -0,0 +1,73 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import multiprocessing
+
+MIN_IMAGES_FOR_PARALLEL = 4
+MAX_WORKERS = multiprocessing.cpu_count()
+
+
+def parallel_downsample_images(base64_strings):
+    import os
+    import sys
+
+    sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
+    from test_result_description.image_processing import (
+        downsample_image,  # type: ignore
+    )
+
+    num_images = len(base64_strings)
+
+    if num_images < MIN_IMAGES_FOR_PARALLEL:
+        return [downsample_image(img) for img in base64_strings]
+
+    num_workers = min(num_images, MAX_WORKERS)
+
+    with multiprocessing.Pool(processes=num_workers) as pool:
+        results = pool.map(downsample_image, base64_strings)
+
+    sys.path.pop(0)
+
+    return results
+
+
+class Context:
+    def __init__(self, mode="local"):
+        pass
+
+    def load(self, input_data):
+        # this task can accept a dict or a test result object from the dev framework
+        if isinstance(input_data, dict):
+            return input_data
+
+        # we are likely running outside of the dev framework and need to convert
+        # the test result object to a dictionary
+        test_result = input_data
+
+        try:
+            from markdownify import markdownify as md
+        except ImportError as e:
+            raise ImportError(
+                "Failed to import markdownify. Please install the package to use this task."
+            ) from e
+
+        input_data = {
+            "test_name": test_result.result_id.split(".")[-1],
+            "test_description": md(test_result.result_metadata[0]["text"]),
+        }
+
+        if hasattr(test_result, "metric") and test_result.metric.summary is not None:
+            input_data["summary"] = test_result.metric.summary.serialize()
+        elif (
+            hasattr(test_result, "test_results")
+            and test_result.test_results.summary is not None
+        ):
+            input_data["summary"] = test_result.test_results.summary.serialize()
+
+        if test_result.figures:
+            input_data["figures"] = parallel_downsample_images(
+                [figure._get_b64_url() for figure in test_result.figures]
+            )
+
+        return input_data

validmind-2.5.18/validmind/ai/test_result_description/image_processing.py

@@ -0,0 +1,124 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import base64
+import io
+
+import numpy as np
+from PIL import Image, ImageEnhance, ImageFilter
+
+DOWNSAMPLE_PERCENTAGE = 50
+
+
+def open_base64_image(base64_string):
+    if base64_string.startswith("data:image/png;base64,"):
+        base64_string = base64_string.split(",")[1]
+
+    image_data = base64.b64decode(base64_string)
+    image_buffer = io.BytesIO(image_data)
+    image = Image.open(image_buffer)
+
+    return image
+
+
+def downsample_image(base64_string):
+    image = open_base64_image(base64_string)
+
+    # Calculate the target dimensions based on the reduction percentage
+    target_width = int(image.width * (1 - DOWNSAMPLE_PERCENTAGE / 100))
+    target_height = int(image.height * (1 - DOWNSAMPLE_PERCENTAGE / 100))
+
+    # If the image is already smaller than the target size, return the original
+    if image.width <= target_width and image.height <= target_height:
+        return base64_string
+
+    # remove any margins from the image
+    # Find the bounding box of non-uniform pixels (margin detection)
+    width, height = image.size
+    background = image.getpixel((0, 0))  # Assume top-left pixel is background color
+
+    def is_different(pixel):
+        return pixel != background
+
+    left = next(
+        x
+        for x in range(width)
+        if any(is_different(image.getpixel((x, y))) for y in range(height))
+    )
+    right = next(
+        x
+        for x in range(width - 1, -1, -1)
+        if any(is_different(image.getpixel((x, y))) for y in range(height))
+    )
+    top = next(
+        y
+        for y in range(height)
+        if any(is_different(image.getpixel((x, y))) for x in range(width))
+    )
+    bottom = next(
+        y
+        for y in range(height - 1, -1, -1)
+        if any(is_different(image.getpixel((x, y))) for x in range(width))
+    )
+
+    # Crop the image to remove the uniform margin (with some padding)
+    bbox = (left - 5, top - 5, right + 6, bottom + 6)
+    image = image.crop(bbox)
+
+    # If the image has an alpha channel, remove any transparent margins
+    if image.mode in ("RGBA", "LA"):
+        alpha = image.getchannel("A")
+        bbox = alpha.getbbox()
+        if bbox:
+            image = image.crop(bbox)
+
+    # Apply unsharp mask to enhance edges
+    image = image.filter(ImageFilter.UnsharpMask(radius=2, percent=150, threshold=3))
+
+    # Calculate new dimensions
+    aspect_ratio = image.width / image.height
+    new_height = target_height
+    new_width = int(new_height * aspect_ratio)
+
+    # print(f"downsampling from {width}x{height} to {new_width}x{new_height}")
+
+    # Ensure we don't exceed the target width
+    if new_width > target_width:
+        new_width = target_width
+        new_height = int(new_width / aspect_ratio)
+
+    # print(f"downsampling from {image.width}x{image.height} to {new_width}x{new_height}")
+
+    # Convert to numpy array for custom downsampling
+    img_array = np.array(image)
+
+    # Optimized area interpolation
+    h_factor = img_array.shape[0] / new_height
+    w_factor = img_array.shape[1] / new_width
+
+    h_indices = (np.arange(new_height).reshape(-1, 1) * h_factor).astype(int)
+    w_indices = (np.arange(new_width).reshape(1, -1) * w_factor).astype(int)
+
+    h_indices = np.minimum(h_indices, img_array.shape[0] - 1)
+    w_indices = np.minimum(w_indices, img_array.shape[1] - 1)
+
+    # Convert back to PIL Image
+    image = Image.fromarray(img_array[h_indices, w_indices].astype(np.uint8))
+
+    # Enhance contrast slightly
+    enhancer = ImageEnhance.Contrast(image)
+    image = enhancer.enhance(1.2)
+
+    # Sharpen the image
+    image = image.filter(ImageFilter.SHARPEN)
+
+    # Convert the image to bytes in PNG format
+    buffered = io.BytesIO()
+    image.save(buffered, format="PNG")
+    img_bytes = buffered.getvalue()
+
+    # Encode the bytes to base64
+    b64_encoded = base64.b64encode(img_bytes).decode("utf-8")
+
+    return f"data:image/png;base64,{b64_encoded}"

validmind-2.5.18/validmind/ai/test_result_description/system.jinja

@@ -0,0 +1,39 @@
+You are an expert data scientist and MRM specialist.
+You are tasked with analyzing the results of a quantitative test run on some model or dataset.
+Your goal is to create a test description that will act as part of the model documentation.
+You will provide both the developer and other consumers of the documentation with a clear and concise "interpretation" of the results they will see.
+The overarching theme to maintain is MRM documentation.
+
+Examine the provided statistical test results and compose a description of the results.
+The results are either in the form of serialized tables or images of plots.
+Compose a description and interpretation of the result to accompany it in MRM documentation.
+It will be read by other data scientists and developers and by validators and stakeholders.
+
+Use valid Markdown syntax to format the response.
+Avoid long sentences and complex vocabulary.
+Avoid overly verbose explanations - the goal is to explain to a user what they are seeing in the results.
+Structure the response clearly and logically.
+Respond only with your analysis and insights, not the verbatim test results.
+Respond only with the markdown content, no explanation or context for your response is necessary.
+Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
+
+Explain the test, its purpose, its mechanism/formula etc and why it is useful.
+If relevant, provide a very brief description of the way this test is used in model/dataset evaluation and how it is interpreted.
+Highlight the key insights from the test results. The key insights should be concise and easily understood.
+An insight should only be included if it is something not entirely obvious from the test results.
+End the response with any closing remarks, summary or additional useful information.
+
+Use the following format for the response (feel free to stray from it if necessary - this is a suggested starting point):
+
+<ResponseFormat>
+**<Test Name>** calculates the xyz <continue to explain what it does in detail>...
+
+This test is useful for <explain why and for what this test is useful>...
+
+**Key Insights:**
+
+The following key insights can be identified in the test results:
+
+- **<key insight 1 - title>**: <concise explanation of key insight 1>
+- ...<continue with any other key insights using the same format>
+</ResponseFormat>

validmind-2.5.18/validmind/ai/test_result_description/user.jinja

@@ -0,0 +1,25 @@
+**Test ID**: `{{ test_name }}`
+
+**Test Description**:
+
+{{ test_description }}
+
+---
+
+Generate a description of the following result of the test using the instructions given in your system prompt.
+
+{%- if summary %}
+**Test Result Tables** *(Raw Data)*:
+{{ summary }}
+{%- endif %}
+
+{%- if figures %}
+The following images make up the results of the test.
+{%- for b64_image_url in figures %}
+[[IMAGE:{{ b64_image_url }}]]
+{%- endfor %}
+{%- endif %}
+
+Keep your response concise and to the point!
+Only include content in your response if its something truly insightful or interesting!
+DO NOT VERBOSELY EXPLAIN THE TEST OR THE RESULTS!!!