validmind 2.5.15__py3-none-any.whl → 2.5.19__py3-none-any.whl

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.5.15"
+__version__ = "2.5.19"

validmind/ai/test_descriptions.py

@@ -3,15 +3,19 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import os
+import re
 from concurrent.futures import ThreadPoolExecutor
 from typing import Union
 
+from jinja2 import Template
+
 from validmind.utils import md_to_html
 
 from ..client_config import client_config
 from ..logging import get_logger
 
 __executor = ThreadPoolExecutor()
+__prompt = None
 
 logger = get_logger(__name__)
 
@@ -19,71 +23,48 @@ logger = get_logger(__name__)
 AI_REVISION_NAME = "Generated by ValidMind AI"
 DEFAULT_REVISION_NAME = "Default Description"
 
-SYSTEM_PROMPT = """  # noqa
-You are an expert data scientist and MRM specialist.
-You are tasked with analyzing the results of a quantitative test run on some model or dataset.
-Your goal is to create a test description that will act as part of the model documentation.
-You will provide both the developer and other consumers of the documentation with a clear and concise "interpretation" of the results they will see.
-The overarching theme to maintain is MRM documentation.
-
-Examine the provided statistical test results and compose a description of the results.
-The results are either in the form of serialized tables or images of plots.
-Compose a description and interpretation of the result to accompany it in MRM documentation.
-It will be read by other data scientists and developers and by validators and stakeholders.
-
-Use valid Markdown syntax to format the response.
-Avoid long sentences and complex vocabulary.
-Avoid overly verbose explanations - the goal is to explain to a user what they are seeing in the results.
-Structure the response clearly and logically.
-Respond only with your analysis and insights, not the verbatim test results.
-Respond only with the markdown content, no explanation or context for your response is necessary.
-Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
-
-Explain the test, its purpose, its mechanism/formula etc and why it is useful.
-If relevant, provide a very brief description of the way this test is used in model/dataset evaluation and how it is interpreted.
-Highlight the key insights from the test results. The key insights should be concise and easily understood.
-An insight should only be included if it is something not entirely obvious from the test results.
-End the response with any closing remarks, summary or additional useful information.
 
-Use the following format for the response (feel free to stray from it if necessary - this is a suggested starting point):
+def _load_prompt():
+    global __prompt
 
-<ResponseFormat>
-**<Test Name>** calculates the xyz <continue to explain what it does in detail>...
+    if not __prompt:
+        folder_path = os.path.join(os.path.dirname(__file__), "test_result_description")
+        with open(os.path.join(folder_path, "system.jinja"), "r") as f:
+            system_prompt = f.read()
+        with open(os.path.join(folder_path, "user.jinja"), "r") as f:
+            user_prompt = f.read()
 
-This test is useful for <explain why and for what this test is useful>...
+        __prompt = (Template(system_prompt), Template(user_prompt))
 
-**Key Insights:**
+    return __prompt
 
-The following key insights can be identified in the test results:
 
-- **<key insight 1 - title>**: <concise explanation of key insight 1>
-- ...<continue with any other key insights using the same format>
-</ResponseFormat>
-""".strip()
+def prompt_to_message(role, prompt):
+    if "[[IMAGE:" not in prompt:
+        return {"role": role, "content": prompt}
 
+    content = []
 
-USER_PROMPT = """
-Test ID: `{test_name}`
+    # Regex pattern to find [[IMAGE:<b64-data>]] markers
+    pattern = re.compile(r"\[\[IMAGE:(.*?)\]\]", re.DOTALL)
 
-<Test Docstring>
-{test_description}
-</Test Docstring>
+    last_index = 0
+    for match in pattern.finditer(prompt):
+        # Text before the image marker
+        start, end = match.span()
+        if start > last_index:
+            content.append({"type": "text", "text": prompt[last_index:start]})
 
-<Test Results Table(s)>
-{test_summary}
-</Test Results Table(s)>
-""".strip()
+        # Image
+        content.append({"type": "image_url", "image_url": {"url": match.group(1)}})
 
+        last_index = end
 
-USER_PROMPT_FIGURES = """
-Test ID: `{test_name}`
+    # Text after the last image
+    if last_index < len(prompt):
+        content.append({"type": "text", "text": prompt[last_index:]})
 
-<Test Docstring>
-{test_description}
-</Test Docstring>
-
-The attached plots show the results of the test.
-""".strip()
+    return {"role": role, "content": content}
 
 
 class DescriptionFuture:
@@ -117,7 +98,9 @@ def generate_description(
 ):
     """Generate the description for the test results"""
     if not test_summary and not figures and not metric:
-        raise ValueError("No summary or figures provided - cannot generate description")
+        raise ValueError(
+            "No summary, unit metric or figures provided - cannot generate description"
+        )
 
     # TODO: fix circular import
     from validmind.ai.utils import get_client_and_model
@@ -140,68 +123,27 @@ def generate_description(
         else:
             test_summary = metric_summary
 
-    if test_summary:
-        logger.debug(
-            f"Generating description for test {test_name} with stringified summary"
-        )
-        return (
-            client.chat.completions.create(
-                model=model,
-                temperature=0,
-                seed=42,
-                messages=[
-                    {"role": "system", "content": SYSTEM_PROMPT},
-                    {
-                        "role": "user",
-                        "content": USER_PROMPT.format(
-                            test_name=test_name,
-                            test_description=test_description,
-                            test_summary=test_summary,
-                        ),
-                    },
-                ],
-            )
-            .choices[0]
-            .message.content.strip()
-        )
+    figures = [] if test_summary else figures
 
-    logger.debug(
-        f"Generating description for test {test_name} with {len(figures)} figures"
-    )
-    return (
-        client.chat.completions.create(
-            model=model,
-            temperature=0,
-            seed=42,
-            messages=[
-                {"role": "system", "content": SYSTEM_PROMPT},
-                {
-                    "role": "user",
-                    "content": [
-                        {
-                            "type": "text",
-                            "text": USER_PROMPT_FIGURES.format(
-                                test_name=test_name,
-                                test_description=test_description,
-                            ),
-                        },
-                        *[
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": figure._get_b64_url(),
-                                },
-                            }
-                            for figure in figures
-                        ],
-                    ],
-                },
-            ],
-        )
-        .choices[0]
-        .message.content.strip()
+    input_data = {
+        "test_name": test_name,
+        "test_description": test_description,
+        "summary": test_summary,
+        "figures": [figure._get_b64_url() for figure in figures],
+    }
+    system, user = _load_prompt()
+
+    response = client.chat.completions.create(
+        model=model,
+        temperature=0.0,
+        messages=[
+            prompt_to_message("system", system.render(input_data)),
+            prompt_to_message("user", user.render(input_data)),
+        ],
     )
 
+    return response.choices[0].message.content
+
 
 def background_generate_description(
     test_id: str,

validmind/ai/test_result_description/config.yaml

@@ -0,0 +1,29 @@
+id: test_result_description
+name: Test Result Description
+description: Generate a description for a test result
+version: 0.1.0
+model: gpt-4o
+temperature: 0.0
+output_type: markdown
+prompts:
+  system:
+    role: system
+    path: system.jinja
+  user:
+    role: user
+    path: user.jinja
+inputs:
+  test_name:
+    description: The name of the test that produced the result (usually the last part of the test ID)
+    type: string
+  test_description:
+    description: The description (docstring) of the test that was run
+    type: string
+  summary:
+    description: The json result summary (i.e. the table(s) returned by the test)
+    type: list
+    optional: true
+  figures:
+    description: A list of base64 encoded images of the figures returned by the test
+    type: list
+    optional: true

validmind/ai/test_result_description/context.py

@@ -0,0 +1,73 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import multiprocessing
+
+MIN_IMAGES_FOR_PARALLEL = 4
+MAX_WORKERS = multiprocessing.cpu_count()
+
+
+def parallel_downsample_images(base64_strings):
+    import os
+    import sys
+
+    sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
+    from test_result_description.image_processing import (
+        downsample_image,  # type: ignore
+    )
+
+    num_images = len(base64_strings)
+
+    if num_images < MIN_IMAGES_FOR_PARALLEL:
+        return [downsample_image(img) for img in base64_strings]
+
+    num_workers = min(num_images, MAX_WORKERS)
+
+    with multiprocessing.Pool(processes=num_workers) as pool:
+        results = pool.map(downsample_image, base64_strings)
+
+    sys.path.pop(0)
+
+    return results
+
+
+class Context:
+    def __init__(self, mode="local"):
+        pass
+
+    def load(self, input_data):
+        # this task can accept a dict or a test result object from the dev framework
+        if isinstance(input_data, dict):
+            return input_data
+
+        # we are likely running outside of the dev framework and need to convert
+        # the test result object to a dictionary
+        test_result = input_data
+
+        try:
+            from markdownify import markdownify as md
+        except ImportError as e:
+            raise ImportError(
+                "Failed to import markdownify. Please install the package to use this task."
+            ) from e
+
+        input_data = {
+            "test_name": test_result.result_id.split(".")[-1],
+            "test_description": md(test_result.result_metadata[0]["text"]),
+        }
+
+        if hasattr(test_result, "metric") and test_result.metric.summary is not None:
+            input_data["summary"] = test_result.metric.summary.serialize()
+        elif (
+            hasattr(test_result, "test_results")
+            and test_result.test_results.summary is not None
+        ):
+            input_data["summary"] = test_result.test_results.summary.serialize()
+
+        if test_result.figures:
+            input_data["figures"] = parallel_downsample_images(
+                [figure._get_b64_url() for figure in test_result.figures]
+            )
+
+        return input_data

validmind/ai/test_result_description/image_processing.py

@@ -0,0 +1,124 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import base64
+import io
+
+import numpy as np
+from PIL import Image, ImageEnhance, ImageFilter
+
+DOWNSAMPLE_PERCENTAGE = 50
+
+
+def open_base64_image(base64_string):
+    if base64_string.startswith("data:image/png;base64,"):
+        base64_string = base64_string.split(",")[1]
+
+    image_data = base64.b64decode(base64_string)
+    image_buffer = io.BytesIO(image_data)
+    image = Image.open(image_buffer)
+
+    return image
+
+
+def downsample_image(base64_string):
+    image = open_base64_image(base64_string)
+
+    # Calculate the target dimensions based on the reduction percentage
+    target_width = int(image.width * (1 - DOWNSAMPLE_PERCENTAGE / 100))
+    target_height = int(image.height * (1 - DOWNSAMPLE_PERCENTAGE / 100))
+
+    # If the image is already smaller than the target size, return the original
+    if image.width <= target_width and image.height <= target_height:
+        return base64_string
+
+    # remove any margins from the image
+    # Find the bounding box of non-uniform pixels (margin detection)
+    width, height = image.size
+    background = image.getpixel((0, 0))  # Assume top-left pixel is background color
+
+    def is_different(pixel):
+        return pixel != background
+
+    left = next(
+        x
+        for x in range(width)
+        if any(is_different(image.getpixel((x, y))) for y in range(height))
+    )
+    right = next(
+        x
+        for x in range(width - 1, -1, -1)
+        if any(is_different(image.getpixel((x, y))) for y in range(height))
+    )
+    top = next(
+        y
+        for y in range(height)
+        if any(is_different(image.getpixel((x, y))) for x in range(width))
+    )
+    bottom = next(
+        y
+        for y in range(height - 1, -1, -1)
+        if any(is_different(image.getpixel((x, y))) for x in range(width))
+    )
+
+    # Crop the image to remove the uniform margin (with some padding)
+    bbox = (left - 5, top - 5, right + 6, bottom + 6)
+    image = image.crop(bbox)
+
+    # If the image has an alpha channel, remove any transparent margins
+    if image.mode in ("RGBA", "LA"):
+        alpha = image.getchannel("A")
+        bbox = alpha.getbbox()
+        if bbox:
+            image = image.crop(bbox)
+
+    # Apply unsharp mask to enhance edges
+    image = image.filter(ImageFilter.UnsharpMask(radius=2, percent=150, threshold=3))
+
+    # Calculate new dimensions
+    aspect_ratio = image.width / image.height
+    new_height = target_height
+    new_width = int(new_height * aspect_ratio)
+
+    # print(f"downsampling from {width}x{height} to {new_width}x{new_height}")
+
+    # Ensure we don't exceed the target width
+    if new_width > target_width:
+        new_width = target_width
+        new_height = int(new_width / aspect_ratio)
+
+    # print(f"downsampling from {image.width}x{image.height} to {new_width}x{new_height}")
+
+    # Convert to numpy array for custom downsampling
+    img_array = np.array(image)
+
+    # Optimized area interpolation
+    h_factor = img_array.shape[0] / new_height
+    w_factor = img_array.shape[1] / new_width
+
+    h_indices = (np.arange(new_height).reshape(-1, 1) * h_factor).astype(int)
+    w_indices = (np.arange(new_width).reshape(1, -1) * w_factor).astype(int)
+
+    h_indices = np.minimum(h_indices, img_array.shape[0] - 1)
+    w_indices = np.minimum(w_indices, img_array.shape[1] - 1)
+
+    # Convert back to PIL Image
+    image = Image.fromarray(img_array[h_indices, w_indices].astype(np.uint8))
+
+    # Enhance contrast slightly
+    enhancer = ImageEnhance.Contrast(image)
+    image = enhancer.enhance(1.2)
+
+    # Sharpen the image
+    image = image.filter(ImageFilter.SHARPEN)
+
+    # Convert the image to bytes in PNG format
+    buffered = io.BytesIO()
+    image.save(buffered, format="PNG")
+    img_bytes = buffered.getvalue()
+
+    # Encode the bytes to base64
+    b64_encoded = base64.b64encode(img_bytes).decode("utf-8")
+
+    return f"data:image/png;base64,{b64_encoded}"

validmind/ai/test_result_description/system.jinja

@@ -0,0 +1,39 @@
+You are an expert data scientist and MRM specialist.
+You are tasked with analyzing the results of a quantitative test run on some model or dataset.
+Your goal is to create a test description that will act as part of the model documentation.
+You will provide both the developer and other consumers of the documentation with a clear and concise "interpretation" of the results they will see.
+The overarching theme to maintain is MRM documentation.
+
+Examine the provided statistical test results and compose a description of the results.
+The results are either in the form of serialized tables or images of plots.
+Compose a description and interpretation of the result to accompany it in MRM documentation.
+It will be read by other data scientists and developers and by validators and stakeholders.
+
+Use valid Markdown syntax to format the response.
+Avoid long sentences and complex vocabulary.
+Avoid overly verbose explanations - the goal is to explain to a user what they are seeing in the results.
+Structure the response clearly and logically.
+Respond only with your analysis and insights, not the verbatim test results.
+Respond only with the markdown content, no explanation or context for your response is necessary.
+Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
+
+Explain the test, its purpose, its mechanism/formula etc and why it is useful.
+If relevant, provide a very brief description of the way this test is used in model/dataset evaluation and how it is interpreted.
+Highlight the key insights from the test results. The key insights should be concise and easily understood.
+An insight should only be included if it is something not entirely obvious from the test results.
+End the response with any closing remarks, summary or additional useful information.
+
+Use the following format for the response (feel free to stray from it if necessary - this is a suggested starting point):
+
+<ResponseFormat>
+**<Test Name>** calculates the xyz <continue to explain what it does in detail>...
+
+This test is useful for <explain why and for what this test is useful>...
+
+**Key Insights:**
+
+The following key insights can be identified in the test results:
+
+- **<key insight 1 - title>**: <concise explanation of key insight 1>
+- ...<continue with any other key insights using the same format>
+</ResponseFormat>

validmind/ai/test_result_description/user.jinja

@@ -0,0 +1,25 @@
+**Test ID**: `{{ test_name }}`
+
+**Test Description**:
+
+{{ test_description }}
+
+---
+
+Generate a description of the following result of the test using the instructions given in your system prompt.
+
+{%- if summary %}
+**Test Result Tables** *(Raw Data)*:
+{{ summary }}
+{%- endif %}
+
+{%- if figures %}
+The following images make up the results of the test.
+{%- for b64_image_url in figures %}
+[[IMAGE:{{ b64_image_url }}]]
+{%- endfor %}
+{%- endif %}
+
+Keep your response concise and to the point!
+Only include content in your response if its something truly insightful or interesting!
+DO NOT VERBOSELY EXPLAIN THE TEST OR THE RESULTS!!!

validmind/datasets/credit_risk/__init__.py

@@ -8,4 +8,5 @@ Entrypoint for credit risk datasets.
 
 __all__ = [
     "lending_club",
+    "lending_club_bias",
 ]

validmind/datasets/credit_risk/lending_club_bias.py

@@ -0,0 +1,142 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import os
+
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import LabelEncoder
+
+current_path = os.path.dirname(os.path.abspath(__file__))
+dataset_path = os.path.join(current_path, "datasets")
+
+# URLs or file paths for online and offline data
+data_file = os.path.join(dataset_path, "lending_club_biased.csv.gz")
+
+target_column = "loan_status"
+protected_classes = ["Gender", "Race", "Marital_Status"]
+
+drop_columns = ["total_pymnt", "id", "verification_status", "purpose"]
+
+score_params = {
+    "target_score": 600,
+    "target_odds": 50,
+    "pdo": 20,
+}
+
+
+def load_data():
+    """
+    Load data from the specified CSV file.
+
+    :return: DataFrame containing the loaded data.
+    """
+
+    print(f"Loading data from: {data_file}")
+    # Since we know the offline_data_file path ends with '.zip', we replace it with '.csv.gz'
+    gzip_file_path = data_file.replace(".zip", ".csv.gz")
+    # Read the CSV file directly from the .gz archive
+    df = pd.read_csv(gzip_file_path, compression="gzip")
+    print("Data loaded successfully.")
+    df = _clean_data(df)
+
+    return df
+
+
+def _clean_data(df):
+    df = df.copy()
+    print("Loading the raw dataset:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Drop columns not relevant for this model
+    print(f"Dropping columns not relevant for this model: {drop_columns}")
+    df = df.drop(columns=drop_columns)
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Drop rows with missing target values
+    df.dropna(subset=[target_column], inplace=True)
+    print("Dropping rows with missing target values:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Drop columns with more than N percent missing values
+    missing_values = df.isnull().mean()
+    df = df.loc[:, missing_values < 0.7]
+    print("Dropping columns with more than 70% missing values:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    # Drop columns with only one unique value
+    unique_values = df.nunique()
+    df = df.loc[:, unique_values > 1]
+    print("Dropping columns with only one unique value:")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    return df
+
+
+def preprocess(df):
+    df = df.copy()
+
+    # Convert the target variable to integer type for modeling.
+    df[target_column] = df[target_column].astype(int)
+
+    # Identify and encode categorical variables for modeling purposes
+    label_encoders = {}
+    categorical_columns = df.select_dtypes(include=["object"]).columns
+
+    for column in categorical_columns:
+        le = LabelEncoder()
+        df[f"{column}_encoded"] = le.fit_transform(df[column])
+        label_encoders[column] = le
+        df = df.drop(columns=[column])  # Remove the original column
+
+    print(f"Encoding categorical variables: {list(categorical_columns)}")
+    print(
+        f"Rows: {df.shape[0]}\nColumns: {df.shape[1]}\nMissing values: {df.isnull().sum().sum()}\n"
+    )
+
+    return df
+
+
+def split(df, test_size=0.3):
+    df = df.copy()
+
+    # Splitting the dataset into training and test sets
+    train_df, test_df = train_test_split(df, test_size=test_size, random_state=42)
+
+    # Calculate and print details for the training dataset
+    print(
+        f"Training Dataset:\nRows: {train_df.shape[0]}\nColumns: {train_df.shape[1]}\nMissing values: {train_df.isnull().sum().sum()}\n"
+    )
+
+    # Calculate and print details for the test dataset
+    print(
+        f"Test Dataset:\nRows: {test_df.shape[0]}\nColumns: {test_df.shape[1]}\nMissing values: {test_df.isnull().sum().sum()}\n"
+    )
+
+    return train_df, test_df
+
+
+def compute_scores(probabilities):
+
+    target_score = score_params["target_score"]
+    target_odds = score_params["target_odds"]
+    pdo = score_params["pdo"]
+
+    factor = pdo / np.log(2)
+    offset = target_score - (factor * np.log(target_odds))
+
+    scores = offset + factor * np.log(probabilities / (1 - probabilities))
+
+    return scores