validmind 2.5.8__py3-none-any.whl → 2.5.18__py3-none-any.whl

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.5.8"
+__version__ = "2.5.18"

validmind/ai/test_descriptions.py

@@ -3,13 +3,19 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import os
+import re
 from concurrent.futures import ThreadPoolExecutor
+from typing import Union
+
+from jinja2 import Template
 
 from validmind.utils import md_to_html
 
+from ..client_config import client_config
 from ..logging import get_logger
 
 __executor = ThreadPoolExecutor()
+__prompt = None
 
 logger = get_logger(__name__)
 
@@ -17,71 +23,48 @@ logger = get_logger(__name__)
 AI_REVISION_NAME = "Generated by ValidMind AI"
 DEFAULT_REVISION_NAME = "Default Description"
 
-SYSTEM_PROMPT = """  # noqa
-You are an expert data scientist and MRM specialist.
-You are tasked with analyzing the results of a quantitative test run on some model or dataset.
-Your goal is to create a test description that will act as part of the model documentation.
-You will provide both the developer and other consumers of the documentation with a clear and concise "interpretation" of the results they will see.
-The overarching theme to maintain is MRM documentation.
-
-Examine the provided statistical test results and compose a description of the results.
-The results are either in the form of serialized tables or images of plots.
-Compose a description and interpretation of the result to accompany it in MRM documentation.
-It will be read by other data scientists and developers and by validators and stakeholders.
-
-Use valid Markdown syntax to format the response.
-Avoid long sentences and complex vocabulary.
-Avoid overly verbose explanations - the goal is to explain to a user what they are seeing in the results.
-Structure the response clearly and logically.
-Respond only with your analysis and insights, not the verbatim test results.
-Respond only with the markdown content, no explanation or context for your response is necessary.
-Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
-
-Explain the test, its purpose, its mechanism/formula etc and why it is useful.
-If relevant, provide a very brief description of the way this test is used in model/dataset evaluation and how it is interpreted.
-Highlight the key insights from the test results. The key insights should be concise and easily understood.
-An insight should only be included if it is something not entirely obvious from the test results.
-End the response with any closing remarks, summary or additional useful information.
 
-Use the following format for the response (feel free to stray from it if necessary - this is a suggested starting point):
+def _load_prompt():
+    global __prompt
 
-<ResponseFormat>
-**<Test Name>** calculates the xyz <continue to explain what it does in detail>...
+    if not __prompt:
+        folder_path = os.path.join(os.path.dirname(__file__), "test_result_description")
+        with open(os.path.join(folder_path, "system.jinja"), "r") as f:
+            system_prompt = f.read()
+        with open(os.path.join(folder_path, "user.jinja"), "r") as f:
+            user_prompt = f.read()
 
-This test is useful for <explain why and for what this test is useful>...
+        __prompt = (Template(system_prompt), Template(user_prompt))
 
-**Key Insights:**
+    return __prompt
 
-The following key insights can be identified in the test results:
 
-- **<key insight 1 - title>**: <concise explanation of key insight 1>
-- ...<continue with any other key insights using the same format>
-</ResponseFormat>
-""".strip()
+def prompt_to_message(role, prompt):
+    if "[[IMAGE:" not in prompt:
+        return {"role": role, "content": prompt}
 
+    content = []
 
-USER_PROMPT = """
-Test ID: `{test_name}`
+    # Regex pattern to find [[IMAGE:<b64-data>]] markers
+    pattern = re.compile(r"\[\[IMAGE:(.*?)\]\]", re.DOTALL)
 
-<Test Docstring>
-{test_description}
-</Test Docstring>
+    last_index = 0
+    for match in pattern.finditer(prompt):
+        # Text before the image marker
+        start, end = match.span()
+        if start > last_index:
+            content.append({"type": "text", "text": prompt[last_index:start]})
 
-<Test Results Table(s)>
-{test_summary}
-</Test Results Table(s)>
-""".strip()
+        # Image
+        content.append({"type": "image_url", "image_url": {"url": match.group(1)}})
 
+        last_index = end
 
-USER_PROMPT_FIGURES = """
-Test ID: `{test_name}`
+    # Text after the last image
+    if last_index < len(prompt):
+        content.append({"type": "text", "text": prompt[last_index:]})
 
-<Test Docstring>
-{test_description}
-</Test Docstring>
-
-The attached plots show the results of the test.
-""".strip()
+    return {"role": role, "content": content}
 
 
 class DescriptionFuture:
@@ -110,11 +93,14 @@ def generate_description(
     test_id: str,
     test_description: str,
     test_summary: str,
+    metric: Union[float, int] = None,
     figures: list = None,
 ):
     """Generate the description for the test results"""
-    if not test_summary and not figures:
-        raise ValueError("No summary or figures provided - cannot generate description")
+    if not test_summary and not figures and not metric:
+        raise ValueError(
+            "No summary, unit metric or figures provided - cannot generate description"
+        )
 
     # TODO: fix circular import
     from validmind.ai.utils import get_client_and_model
@@ -130,79 +116,50 @@ def generate_description(
         else test_description
     )
 
-    if test_summary:
-        logger.debug(
-            f"Generating description for test {test_name} with stringified summary"
-        )
-        return (
-            client.chat.completions.create(
-                model=model,
-                temperature=0,
-                seed=42,
-                messages=[
-                    {"role": "system", "content": SYSTEM_PROMPT},
-                    {
-                        "role": "user",
-                        "content": USER_PROMPT.format(
-                            test_name=test_name,
-                            test_description=test_description,
-                            test_summary=test_summary,
-                        ),
-                    },
-                ],
-            )
-            .choices[0]
-            .message.content.strip()
-        )
+    if metric:
+        metric_summary = f"**Metric Value**: {metric}"
+        if test_summary:
+            test_summary = metric_summary + "\n" + test_summary
+        else:
+            test_summary = metric_summary
 
-    logger.debug(
-        f"Generating description for test {test_name} with {len(figures)} figures"
-    )
-    return (
-        client.chat.completions.create(
-            model=model,
-            temperature=0,
-            seed=42,
-            messages=[
-                {"role": "system", "content": SYSTEM_PROMPT},
-                {
-                    "role": "user",
-                    "content": [
-                        {
-                            "type": "text",
-                            "text": USER_PROMPT_FIGURES.format(
-                                test_name=test_name,
-                                test_description=test_description,
-                            ),
-                        },
-                        *[
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": figure._get_b64_url(),
-                                },
-                            }
-                            for figure in figures
-                        ],
-                    ],
-                },
-            ],
-        )
-        .choices[0]
-        .message.content.strip()
+    figures = [] if test_summary else figures
+
+    input_data = {
+        "test_name": test_name,
+        "test_description": test_description,
+        "summary": test_summary,
+        "figures": [figure._get_b64_url() for figure in figures],
+    }
+    system, user = _load_prompt()
+
+    response = client.chat.completions.create(
+        model=model,
+        temperature=0.0,
+        messages=[
+            prompt_to_message("system", system.render(input_data)),
+            prompt_to_message("user", user.render(input_data)),
+        ],
     )
 
+    return response.choices[0].message.content
+
 
 def background_generate_description(
     test_id: str,
     test_description: str,
     test_summary: str,
     figures: list = None,
+    metric: Union[int, float] = None,
 ):
     def wrapped():
         try:
             return generate_description(
-                test_id, test_description, test_summary, figures
+                test_id=test_id,
+                test_description=test_description,
+                test_summary=test_summary,
+                figures=figures,
+                metric=metric,
             )
         except Exception as e:
             logger.error(f"Failed to generate description: {e}")
@@ -217,6 +174,7 @@ def get_description_metadata(
     default_description,
     summary=None,
     figures=None,
+    metric=None,
     prefix="metric_description",
     should_generate=True,
 ):
@@ -238,16 +196,18 @@ def get_description_metadata(
         default_description (str): The default description for the test
         summary (Any): The test summary or results to interpret
         figures (List[Figure]): The figures to attach to the test suite result
+        metric (Union[int, float]): Unit metrics attached to the test result
         prefix (str): The prefix to use for the content ID (Default: "metric_description")
         should_generate (bool): Whether to generate the description or not (Default: True)
 
     Returns:
         dict: The metadata object to be logged with the test results
     """
-    env_disabled = os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1") in [
-        "0",
-        "false",
-    ]
+    # Check the feature flag first, then the environment variable
+    llm_descriptions_enabled = (
+        client_config.can_generate_llm_test_descriptions()
+        and os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1") not in ["0", "false"]
+    )
 
     # TODO: fix circular import
     from validmind.ai.utils import is_configured
@@ -255,7 +215,7 @@ def get_description_metadata(
     if (
         should_generate
         and (summary or figures)
-        and not env_disabled
+        and llm_descriptions_enabled
         and is_configured()
     ):
         revision_name = AI_REVISION_NAME
@@ -267,6 +227,7 @@ def get_description_metadata(
             test_description=default_description,
             test_summary=summary,
             figures=figures,
+            metric=metric,
         )
 
     else:

validmind/ai/test_result_description/config.yaml

@@ -0,0 +1,29 @@
+id: test_result_description
+name: Test Result Description
+description: Generate a description for a test result
+version: 0.1.0
+model: gpt-4o
+temperature: 0.0
+output_type: markdown
+prompts:
+  system:
+    role: system
+    path: system.jinja
+  user:
+    role: user
+    path: user.jinja
+inputs:
+  test_name:
+    description: The name of the test that produced the result (usually the last part of the test ID)
+    type: string
+  test_description:
+    description: The description (docstring) of the test that was run
+    type: string
+  summary:
+    description: The json result summary (i.e. the table(s) returned by the test)
+    type: list
+    optional: true
+  figures:
+    description: A list of base64 encoded images of the figures returned by the test
+    type: list
+    optional: true

validmind/ai/test_result_description/context.py

@@ -0,0 +1,73 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import multiprocessing
+
+MIN_IMAGES_FOR_PARALLEL = 4
+MAX_WORKERS = multiprocessing.cpu_count()
+
+
+def parallel_downsample_images(base64_strings):
+    import os
+    import sys
+
+    sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
+    from test_result_description.image_processing import (
+        downsample_image,  # type: ignore
+    )
+
+    num_images = len(base64_strings)
+
+    if num_images < MIN_IMAGES_FOR_PARALLEL:
+        return [downsample_image(img) for img in base64_strings]
+
+    num_workers = min(num_images, MAX_WORKERS)
+
+    with multiprocessing.Pool(processes=num_workers) as pool:
+        results = pool.map(downsample_image, base64_strings)
+
+    sys.path.pop(0)
+
+    return results
+
+
+class Context:
+    def __init__(self, mode="local"):
+        pass
+
+    def load(self, input_data):
+        # this task can accept a dict or a test result object from the dev framework
+        if isinstance(input_data, dict):
+            return input_data
+
+        # we are likely running outside of the dev framework and need to convert
+        # the test result object to a dictionary
+        test_result = input_data
+
+        try:
+            from markdownify import markdownify as md
+        except ImportError as e:
+            raise ImportError(
+                "Failed to import markdownify. Please install the package to use this task."
+            ) from e
+
+        input_data = {
+            "test_name": test_result.result_id.split(".")[-1],
+            "test_description": md(test_result.result_metadata[0]["text"]),
+        }
+
+        if hasattr(test_result, "metric") and test_result.metric.summary is not None:
+            input_data["summary"] = test_result.metric.summary.serialize()
+        elif (
+            hasattr(test_result, "test_results")
+            and test_result.test_results.summary is not None
+        ):
+            input_data["summary"] = test_result.test_results.summary.serialize()
+
+        if test_result.figures:
+            input_data["figures"] = parallel_downsample_images(
+                [figure._get_b64_url() for figure in test_result.figures]
+            )
+
+        return input_data

validmind/ai/test_result_description/image_processing.py

@@ -0,0 +1,124 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import base64
+import io
+
+import numpy as np
+from PIL import Image, ImageEnhance, ImageFilter
+
+DOWNSAMPLE_PERCENTAGE = 50
+
+
+def open_base64_image(base64_string):
+    if base64_string.startswith("data:image/png;base64,"):
+        base64_string = base64_string.split(",")[1]
+
+    image_data = base64.b64decode(base64_string)
+    image_buffer = io.BytesIO(image_data)
+    image = Image.open(image_buffer)
+
+    return image
+
+
+def downsample_image(base64_string):
+    image = open_base64_image(base64_string)
+
+    # Calculate the target dimensions based on the reduction percentage
+    target_width = int(image.width * (1 - DOWNSAMPLE_PERCENTAGE / 100))
+    target_height = int(image.height * (1 - DOWNSAMPLE_PERCENTAGE / 100))
+
+    # If the image is already smaller than the target size, return the original
+    if image.width <= target_width and image.height <= target_height:
+        return base64_string
+
+    # remove any margins from the image
+    # Find the bounding box of non-uniform pixels (margin detection)
+    width, height = image.size
+    background = image.getpixel((0, 0))  # Assume top-left pixel is background color
+
+    def is_different(pixel):
+        return pixel != background
+
+    left = next(
+        x
+        for x in range(width)
+        if any(is_different(image.getpixel((x, y))) for y in range(height))
+    )
+    right = next(
+        x
+        for x in range(width - 1, -1, -1)
+        if any(is_different(image.getpixel((x, y))) for y in range(height))
+    )
+    top = next(
+        y
+        for y in range(height)
+        if any(is_different(image.getpixel((x, y))) for x in range(width))
+    )
+    bottom = next(
+        y
+        for y in range(height - 1, -1, -1)
+        if any(is_different(image.getpixel((x, y))) for x in range(width))
+    )
+
+    # Crop the image to remove the uniform margin (with some padding)
+    bbox = (left - 5, top - 5, right + 6, bottom + 6)
+    image = image.crop(bbox)
+
+    # If the image has an alpha channel, remove any transparent margins
+    if image.mode in ("RGBA", "LA"):
+        alpha = image.getchannel("A")
+        bbox = alpha.getbbox()
+        if bbox:
+            image = image.crop(bbox)
+
+    # Apply unsharp mask to enhance edges
+    image = image.filter(ImageFilter.UnsharpMask(radius=2, percent=150, threshold=3))
+
+    # Calculate new dimensions
+    aspect_ratio = image.width / image.height
+    new_height = target_height
+    new_width = int(new_height * aspect_ratio)
+
+    # print(f"downsampling from {width}x{height} to {new_width}x{new_height}")
+
+    # Ensure we don't exceed the target width
+    if new_width > target_width:
+        new_width = target_width
+        new_height = int(new_width / aspect_ratio)
+
+    # print(f"downsampling from {image.width}x{image.height} to {new_width}x{new_height}")
+
+    # Convert to numpy array for custom downsampling
+    img_array = np.array(image)
+
+    # Optimized area interpolation
+    h_factor = img_array.shape[0] / new_height
+    w_factor = img_array.shape[1] / new_width
+
+    h_indices = (np.arange(new_height).reshape(-1, 1) * h_factor).astype(int)
+    w_indices = (np.arange(new_width).reshape(1, -1) * w_factor).astype(int)
+
+    h_indices = np.minimum(h_indices, img_array.shape[0] - 1)
+    w_indices = np.minimum(w_indices, img_array.shape[1] - 1)
+
+    # Convert back to PIL Image
+    image = Image.fromarray(img_array[h_indices, w_indices].astype(np.uint8))
+
+    # Enhance contrast slightly
+    enhancer = ImageEnhance.Contrast(image)
+    image = enhancer.enhance(1.2)
+
+    # Sharpen the image
+    image = image.filter(ImageFilter.SHARPEN)
+
+    # Convert the image to bytes in PNG format
+    buffered = io.BytesIO()
+    image.save(buffered, format="PNG")
+    img_bytes = buffered.getvalue()
+
+    # Encode the bytes to base64
+    b64_encoded = base64.b64encode(img_bytes).decode("utf-8")
+
+    return f"data:image/png;base64,{b64_encoded}"

validmind/ai/test_result_description/system.jinja

@@ -0,0 +1,39 @@
+You are an expert data scientist and MRM specialist.
+You are tasked with analyzing the results of a quantitative test run on some model or dataset.
+Your goal is to create a test description that will act as part of the model documentation.
+You will provide both the developer and other consumers of the documentation with a clear and concise "interpretation" of the results they will see.
+The overarching theme to maintain is MRM documentation.
+
+Examine the provided statistical test results and compose a description of the results.
+The results are either in the form of serialized tables or images of plots.
+Compose a description and interpretation of the result to accompany it in MRM documentation.
+It will be read by other data scientists and developers and by validators and stakeholders.
+
+Use valid Markdown syntax to format the response.
+Avoid long sentences and complex vocabulary.
+Avoid overly verbose explanations - the goal is to explain to a user what they are seeing in the results.
+Structure the response clearly and logically.
+Respond only with your analysis and insights, not the verbatim test results.
+Respond only with the markdown content, no explanation or context for your response is necessary.
+Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
+
+Explain the test, its purpose, its mechanism/formula etc and why it is useful.
+If relevant, provide a very brief description of the way this test is used in model/dataset evaluation and how it is interpreted.
+Highlight the key insights from the test results. The key insights should be concise and easily understood.
+An insight should only be included if it is something not entirely obvious from the test results.
+End the response with any closing remarks, summary or additional useful information.
+
+Use the following format for the response (feel free to stray from it if necessary - this is a suggested starting point):
+
+<ResponseFormat>
+**<Test Name>** calculates the xyz <continue to explain what it does in detail>...
+
+This test is useful for <explain why and for what this test is useful>...
+
+**Key Insights:**
+
+The following key insights can be identified in the test results:
+
+- **<key insight 1 - title>**: <concise explanation of key insight 1>
+- ...<continue with any other key insights using the same format>
+</ResponseFormat>

validmind/ai/test_result_description/user.jinja

@@ -0,0 +1,25 @@
+**Test ID**: `{{ test_name }}`
+
+**Test Description**:
+
+{{ test_description }}
+
+---
+
+Generate a description of the following result of the test using the instructions given in your system prompt.
+
+{%- if summary %}
+**Test Result Tables** *(Raw Data)*:
+{{ summary }}
+{%- endif %}
+
+{%- if figures %}
+The following images make up the results of the test.
+{%- for b64_image_url in figures %}
+[[IMAGE:{{ b64_image_url }}]]
+{%- endfor %}
+{%- endif %}
+
+Keep your response concise and to the point!
+Only include content in your response if its something truly insightful or interesting!
+DO NOT VERBOSELY EXPLAIN THE TEST OR THE RESULTS!!!