validmind 2.7.12__py3-none-any.whl → 2.8.12__py3-none-any.whl

validmind/__init__.py

@@ -30,8 +30,12 @@ vm.init(
 
 After you have pasted the code snippet into your development source code and executed the code, the Python Library API will register with ValidMind. You can now use the ValidMind Library to document and test your models, and to upload to the ValidMind Platform.
 """
+import threading
 import warnings
 
+import pkg_resources
+from IPython.display import HTML, display
+
 # Ignore Numba warnings. We are not requiring this package directly
 from numba.core.errors import NumbaDeprecationWarning, NumbaPendingDeprecationWarning
 
@@ -51,30 +55,74 @@ from .client import (  # noqa: E402
 )
 from .tests.decorator import tags, tasks, test
 from .tests.run import print_env
+from .utils import is_notebook, parse_version
 from .vm_models.result import RawData
 
+__shown = False
+
+
+def show_warning(installed, running):
+    global __shown
+
+    if __shown:
+        return
+    __shown = True
+
+    message = (
+        f"⚠️ This kernel is running an older version of validmind ({running}) "
+        f"than the latest version installed on your system ({installed}).\n\n"
+        "You may need to restart the kernel if you are experiencing issues."
+    )
+    display(HTML(f"<div style='color: red;'>{message}</div>"))
+
+
+def check_version():
+    # get the installed vs running version of validmind
+    # to make sure we are using the latest installed version
+    # in case user has updated the package but forgot to restart the kernel
+    installed = pkg_resources.get_distribution("validmind").version
+    running = __version__
+
+    if parse_version(installed) > parse_version(running):
+        show_warning(installed, running)
+
+    # Schedule the next check for 5 minutes from now
+    timer = threading.Timer(300, check_version)
+    timer.daemon = True
+    timer.start()
+
+
+if is_notebook():
+    check_version()
+
 __all__ = [  # noqa
     "__version__",
-    # Python Library API
-    "datasets",
-    "errors",
-    "get_test_suite",
+    # main library API
     "init",
+    "reload",
     "init_dataset",
     "init_model",
     "init_r_model",
     "preview_template",
-    "print_env",
-    "RawData",
-    "reload",
     "run_documentation_tests",
+    # log metric function (for direct/bulk/retroactive logging of metrics)
+    "log_metric",
+    # test suite functions (less common)
+    "get_test_suite",
     "run_test_suite",
+    # helper functions (for troubleshooting)
+    "print_env",
+    # decorators (for building tests
     "tags",
     "tasks",
     "test",
-    "tests",
-    "test_suites",
+    # raw data (for post-processing test results and building tests)
+    "RawData",
+    # submodules
+    "datasets",
+    "errors",
     "vm_models",
+    "tests",
     "unit_metrics",
-    "log_metric",
+    "test_suites",
 ]

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.7.12"
+__version__ = "2.8.12"

validmind/ai/test_descriptions.py

@@ -4,70 +4,24 @@
 
 import json
 import os
-import re
 from concurrent.futures import ThreadPoolExecutor
 from typing import List, Optional, Union
 
 import tiktoken
-from jinja2 import Template
 
 from ..client_config import client_config
 from ..logging import get_logger
 from ..utils import NumpyEncoder, md_to_html, test_id_to_name
 from ..vm_models.figure import Figure
 from ..vm_models.result import ResultTable
-from .utils import DescriptionFuture, get_client_and_model
+from .utils import DescriptionFuture
 
 __executor = ThreadPoolExecutor()
-__prompt = None
 
 logger = get_logger(__name__)
 
 
-def _load_prompt():
-    global __prompt
-
-    if not __prompt:
-        folder_path = os.path.join(os.path.dirname(__file__), "test_result_description")
-        with open(os.path.join(folder_path, "system.jinja"), "r") as f:
-            system_prompt = f.read()
-        with open(os.path.join(folder_path, "user.jinja"), "r") as f:
-            user_prompt = f.read()
-
-        __prompt = (Template(system_prompt), Template(user_prompt))
-
-    return __prompt
-
-
-def prompt_to_message(role, prompt):
-    if "[[IMAGE:" not in prompt:
-        return {"role": role, "content": prompt}
-
-    content = []
-
-    # Regex pattern to find [[IMAGE:<b64-data>]] markers
-    pattern = re.compile(r"\[\[IMAGE:(.*?)\]\]", re.DOTALL)
-
-    last_index = 0
-    for match in pattern.finditer(prompt):
-        # Text before the image marker
-        start, end = match.span()
-        if start > last_index:
-            content.append({"type": "text", "text": prompt[last_index:start]})
-
-        content.append({"type": "image_url", "image_url": {"url": match.group(1)}})
-
-        last_index = end
-
-    # Text after the last image
-    if last_index < len(prompt):
-        content.append({"type": "text", "text": prompt[last_index:]})
-
-    return {"role": role, "content": content}
-
-
 def _get_llm_global_context():
-
     # Get the context from the environment variable
     context = os.getenv("VALIDMIND_LLM_DESCRIPTIONS_CONTEXT", "")
 
@@ -117,13 +71,13 @@ def generate_description(
     title: Optional[str] = None,
 ):
     """Generate the description for the test results"""
+    from validmind.api_client import generate_test_result_description
+
     if not tables and not figures and not metric:
         raise ValueError(
             "No tables, unit metric or figures provided - cannot generate description"
         )
 
-    client, model = get_client_and_model()
-
     # get last part of test id
     test_name = title or test_id.split(".")[-1]
 
@@ -147,29 +101,18 @@ def generate_description(
     else:
         summary = None
 
-    context = _get_llm_global_context()
-
-    input_data = {
-        "test_name": test_name,
-        "test_description": test_description,
-        "title": title,
-        "summary": _truncate_summary(summary, test_id),
-        "figures": [figure._get_b64_url() for figure in ([] if tables else figures)],
-        "context": context,
-    }
-    system, user = _load_prompt()
-
-    messages = [
-        prompt_to_message("system", system.render(input_data)),
-        prompt_to_message("user", user.render(input_data)),
-    ]
-    response = client.chat.completions.create(
-        model=model,
-        temperature=0.0,
-        messages=messages,
-    )
-
-    return response.choices[0].message.content
+    return generate_test_result_description(
+        {
+            "test_name": test_name,
+            "test_description": test_description,
+            "title": title,
+            "summary": _truncate_summary(summary, test_id),
+            "figures": [
+                figure._get_b64_url() for figure in ([] if tables else figures)
+            ],
+            "context": _get_llm_global_context(),
+        }
+    )["content"]
 
 
 def background_generate_description(
@@ -240,7 +183,8 @@ def get_result_description(
     # Check the feature flag first, then the environment variable
     llm_descriptions_enabled = (
         client_config.can_generate_llm_test_descriptions()
-        and os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1") not in ["0", "false"]
+        and os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1").lower()
+        not in ["0", "false"]
     )
 
     # TODO: fix circular import

validmind/api_client.py

@@ -194,6 +194,7 @@ def init(
     api_host: Optional[str] = None,
     model: Optional[str] = None,
     monitoring: bool = False,
+    generate_descriptions: Optional[bool] = None,
 ):
     """
     Initializes the API client instances and calls the /ping endpoint to ensure
@@ -209,7 +210,7 @@ def init(
         api_secret (str, optional): The API secret. Defaults to None.
         api_host (str, optional): The API host. Defaults to None.
         monitoring (bool): The ongoing monitoring flag. Defaults to False.
-
+        generate_descriptions (bool): Whether to use GenAI to generate test result descriptions. Defaults to True.
     Raises:
         ValueError: If the API key and secret are not provided
     """
@@ -235,6 +236,9 @@ def init(
 
     _monitoring = monitoring
 
+    if generate_descriptions is not None:
+        os.environ["VALIDMIND_LLM_DESCRIPTIONS_ENABLED"] = str(generate_descriptions)
+
     reload()
 
 
@@ -487,3 +491,16 @@ def get_ai_key() -> Dict[str, Any]:
         raise_api_error(r.text)
 
     return r.json()
+
+
+def generate_test_result_description(test_result_data: Dict[str, Any]) -> str:
+    r = requests.post(
+        url=_get_url("ai/generate/test_result_description"),
+        headers=_get_api_headers(),
+        json=test_result_data,
+    )
+
+    if r.status_code != 200:
+        raise_api_error(r.text)
+
+    return r.json()

validmind/models/r_model.py

@@ -5,6 +5,7 @@
 import numpy as np
 import pandas as pd
 
+from validmind.errors import MissingRExtrasError
 from validmind.logging import get_logger
 from validmind.vm_models.model import VMModel
 
@@ -125,7 +126,10 @@ class RModel(VMModel):
         """
         Converts the predicted probabilities to classes
         """
-        from rpy2.robjects import pandas2ri
+        try:
+            from rpy2.robjects import pandas2ri
+        except ImportError:
+            raise MissingRExtrasError()
 
         # Activate the pandas conversion for rpy2
         pandas2ri.activate()

validmind/tests/comparison.py

@@ -15,7 +15,7 @@ from validmind.vm_models.figure import (
     is_png_image,
 )
 from validmind.vm_models.input import VMInput
-from validmind.vm_models.result import ResultTable, TestResult
+from validmind.vm_models.result import RawData, ResultTable, TestResult
 
 logger = get_logger(__name__)
 
@@ -312,6 +312,25 @@ def get_comparison_test_configs(
     return test_configs
 
 
+def _combine_raw_data(results: List[TestResult]) -> RawData:
+    """Combine RawData objects"""
+    attribute_names = results[0].raw_data.__dict__.keys()
+
+    # check that all the raw data objects have the same attributes
+    for result in results:
+        if not isinstance(result.raw_data, RawData):
+            raise ValueError("All raw data objects must be of type RawData")
+        if result.raw_data.__dict__.keys() != attribute_names:
+            raise ValueError("RawData objects must have the same attributes")
+
+    return RawData(
+        **{
+            key: [getattr(result.raw_data, key) for result in results]
+            for key in attribute_names
+        }
+    )
+
+
 def combine_results(
     results: List[TestResult],
 ) -> Tuple[List[Any], Dict[str, List[Any]], Dict[str, List[Any]]]:
@@ -338,6 +357,9 @@ def combine_results(
     # handle threshold tests (i.e. tests that have pass/fail bool status)
     if results[0].passed is not None:
         combined_outputs.append(all(result.passed for result in results))
+    # handle raw data (if any)
+    if results[0].raw_data:
+        combined_outputs.append(_combine_raw_data(results))
 
     # combine inputs and params
     combined_inputs = {}
@@ -359,4 +381,8 @@ def combine_results(
     combined_inputs = _combine_dict_values(combined_inputs)
     combined_params = _combine_dict_values(combined_params)
 
-    return combined_outputs, combined_inputs, combined_params
+    return (
+        tuple(combined_outputs),
+        combined_inputs,
+        combined_params,
+    )

validmind/tests/data_validation/ACFandPACFPlot.py

@@ -94,4 +94,7 @@ def ACFandPACFPlot(dataset: VMDataset):
         figures.append(pacf_fig)
         pacf_store[col] = pacf_values
 
-    return (*figures, RawData(acf_values=acf_store, pacf_values=pacf_store))
+    return (
+        *figures,
+        RawData(acf_values=acf_store, pacf_values=pacf_store, dataset=dataset.input_id),
+    )

validmind/tests/data_validation/AutoMA.py

@@ -116,4 +116,4 @@ def AutoMA(dataset: VMDataset, max_ma_order: int = 3):
     return {
         "Auto MA Analysis Results": summary_ma_analysis,
         "Best MA Order Results": best_ma_order,
-    }, RawData(raw_series_data=df)
+    }, RawData(raw_series_data=df, dataset=dataset.input_id)

validmind/tests/data_validation/BivariateScatterPlots.py

@@ -80,5 +80,9 @@ def BivariateScatterPlots(dataset):
         figures.append(fig)
 
     return tuple(figures) + (
-        RawData(selected_numerical_df=df, feature_pairs=features_pairs),
+        RawData(
+            selected_numerical_df=df,
+            feature_pairs=features_pairs,
+            dataset=dataset.input_id,
+        ),
     )

validmind/tests/data_validation/BoxPierce.py

@@ -68,4 +68,6 @@ def BoxPierce(dataset):
     box_pierce_df.reset_index(inplace=True)
     box_pierce_df.columns = ["column", "stat", "pvalue"]
 
-    return box_pierce_df, RawData(box_pierce_values=box_pierce_values)
+    return box_pierce_df, RawData(
+        box_pierce_values=box_pierce_values, dataset=dataset.input_id
+    )

validmind/tests/data_validation/ClassImbalance.py

@@ -104,5 +104,5 @@ def ClassImbalance(
         },
         go.Figure(data=[trace], layout=layout),
         all(row["Pass/Fail"] == "Pass" for row in imbalanced_classes),
-        RawData(imbalance_percentages=imbalance_percentages),
+        RawData(imbalance_percentages=imbalance_percentages, dataset=dataset.input_id),
     )

validmind/tests/data_validation/DatasetDescription.py

@@ -242,4 +242,4 @@ def DatasetDescription(dataset: VMDataset):
             }
             for column in results
         ]
-    }, RawData(raw_data=raw_data)
+    }, RawData(raw_data=raw_data, dataset=dataset.input_id)

validmind/tests/data_validation/DickeyFullerGLS.py

@@ -97,4 +97,4 @@ def DickeyFullerGLS(dataset: VMDataset):
 
     return {
         "DFGLS Test Results": dfgls_values,
-    }, RawData(df=df)
+    }, RawData(df=df, dataset=dataset.input_id)

validmind/tests/data_validation/FeatureTargetCorrelationPlot.py

@@ -52,19 +52,13 @@ def FeatureTargetCorrelationPlot(dataset, fig_height=600):
     - Not apt for models that employ complex feature interactions, like Decision Trees or Neural Networks, as the test
     may not accurately reflect their importance.
     """
-
-    # Filter DataFrame based on features and target_column
     df = dataset.df[dataset.feature_columns + [dataset.target_column]]
 
-    fig = _visualize_feature_target_correlation(df, dataset.target_column, fig_height)
-
-    correlations = (
-        df.corr(numeric_only=True)[dataset.target_column]
-        .drop(dataset.target_column)
-        .to_frame()
+    fig, correlations = _visualize_feature_target_correlation(
+        df, dataset.target_column, fig_height
     )
 
-    return fig, RawData(correlation_data=correlations)
+    return fig, RawData(correlation_data=correlations, dataset=dataset.input_id)
 
 
 def _visualize_feature_target_correlation(df, target_column, fig_height):
@@ -100,4 +94,5 @@ def _visualize_feature_target_correlation(df, target_column, fig_height):
         yaxis_title="",
         height=fig_height,  # Adjust the height value as needed
     )
-    return fig
+
+    return fig, correlations

validmind/tests/data_validation/HighCardinality.py

@@ -83,4 +83,8 @@ def HighCardinality(
         if not passed:
             all_passed = False
 
-    return table, all_passed, RawData(raw_cardinality_details=raw_data)
+    return (
+        table,
+        all_passed,
+        RawData(raw_cardinality_details=raw_data, dataset=dataset.input_id),
+    )

validmind/tests/data_validation/HighPearsonCorrelation.py

@@ -84,5 +84,5 @@ def HighPearsonCorrelation(
     return (
         pairs,
         all(p["Pass/Fail"] == "Pass" for p in pairs),
-        RawData(correlation_matrix=corr),
+        RawData(correlation_matrix=corr, dataset=dataset.input_id),
     )

validmind/tests/data_validation/IQROutliersBarPlot.py

@@ -118,11 +118,13 @@ def IQROutliersBarPlot(
         )
         figures.append(fig)
 
+    outliers_by_feature = df[dataset.feature_columns_numeric].apply(
+        lambda col: compute_outliers(col, threshold)
+    )
+
     return (
         *figures,
         RawData(
-            outlier_counts_by_feature=df[dataset.feature_columns_numeric].apply(
-                lambda col: compute_outliers(col, threshold)
-            )
+            outlier_counts_by_feature=outliers_by_feature, dataset=dataset.input_id
         ),
     )

validmind/tests/data_validation/IQROutliersTable.py

@@ -2,7 +2,7 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset
 
 
@@ -64,6 +64,7 @@ def IQROutliersTable(dataset: VMDataset, threshold: float = 1.5):
     df = dataset.df
 
     outliers_table = []
+    all_outliers = {}
 
     for col in dataset.feature_columns_numeric:
         # Skip binary features
@@ -71,6 +72,8 @@ def IQROutliersTable(dataset: VMDataset, threshold: float = 1.5):
             continue
 
         outliers = compute_outliers(df[col], threshold)
+        all_outliers[col] = outliers
+
         if outliers.empty:
             continue
 
@@ -89,4 +92,4 @@ def IQROutliersTable(dataset: VMDataset, threshold: float = 1.5):
 
     return {
         "Summary of Outliers Detected by IQR Method": outliers_table,
-    }
+    }, RawData(all_outliers=all_outliers, dataset=dataset.input_id)

validmind/tests/data_validation/IsolationForestOutliers.py

@@ -8,7 +8,7 @@ import matplotlib.pyplot as plt
 import seaborn as sns
 from sklearn.ensemble import IsolationForest
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset
 
 
@@ -91,6 +91,7 @@ def IsolationForestOutliers(
 
         figures.append(fig)
 
-        plt.close()
-
-    return tuple(figures)
+    return (
+        *figures,
+        RawData(predictions=y_pred, dataset=dataset.input_id),
+    )

validmind/tests/data_validation/JarqueBera.py

@@ -5,7 +5,7 @@
 import pandas as pd
 from statsmodels.stats.stattools import jarque_bera
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tasks("classification", "regression")
@@ -67,4 +67,4 @@ def JarqueBera(dataset):
     jb_df.reset_index(inplace=True)
     jb_df.columns = ["column", "stat", "pvalue", "skew", "kurtosis"]
 
-    return jb_df
+    return jb_df, RawData(jb_values=jb_values, dataset=dataset.input_id)

validmind/tests/data_validation/LJungBox.py

@@ -5,7 +5,7 @@
 import pandas as pd
 from statsmodels.stats.diagnostic import acorr_ljungbox
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 
 
 @tasks("regression")
@@ -63,4 +63,4 @@ def LJungBox(dataset):
     ljung_box_df.reset_index(inplace=True)
     ljung_box_df.columns = ["column", "stat", "pvalue"]
 
-    return ljung_box_df
+    return ljung_box_df, RawData(ljung_box_df=ljung_box_df, dataset=dataset.input_id)

validmind/tests/data_validation/LaggedCorrelationHeatmap.py

@@ -101,4 +101,4 @@ def LaggedCorrelationHeatmap(dataset: VMDataset, num_lags: int = 10):
         xaxis_title="Lags",
     )
 
-    return fig, RawData(correlation_matrix=correlation_df)
+    return fig, RawData(correlation_matrix=correlation_df, dataset=dataset.input_id)

validmind/tests/data_validation/MissingValues.py

@@ -2,7 +2,7 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.vm_models import VMDataset
 
 
@@ -49,12 +49,16 @@ def MissingValues(dataset: VMDataset, min_threshold: int = 1):
     df = dataset.df
     missing = df.isna().sum()
 
-    return [
-        {
-            "Column": col,
-            "Number of Missing Values": missing[col],
-            "Percentage of Missing Values (%)": missing[col] / df.shape[0] * 100,
-            "Pass/Fail": "Pass" if missing[col] < min_threshold else "Fail",
-        }
-        for col in missing.index
-    ], all(missing[col] < min_threshold for col in missing.index)
+    return (
+        [
+            {
+                "Column": col,
+                "Number of Missing Values": missing[col],
+                "Percentage of Missing Values (%)": missing[col] / df.shape[0] * 100,
+                "Pass/Fail": "Pass" if missing[col] < min_threshold else "Fail",
+            }
+            for col in missing.index
+        ],
+        all(missing[col] < min_threshold for col in missing.index),
+        RawData(missing_values=missing, dataset=dataset.input_id),
+    )

validmind/tests/data_validation/MissingValuesBarPlot.py

@@ -117,5 +117,7 @@ def MissingValuesBarPlot(
                 height=fig_height,
             ),
         ),
-        RawData(missing_percentages=missing_percentages_sorted),
+        RawData(
+            missing_percentages=missing_percentages_sorted, dataset=dataset.input_id
+        ),
     )

validmind/tests/data_validation/MutualInformation.py

@@ -123,5 +123,6 @@ def MutualInformation(
     return fig, RawData(
         mutual_information_scores={
             feature: score for feature, score in zip(sorted_features, sorted_scores)
-        }
+        },
+        dataset=dataset.input_id,
     )

validmind/tests/data_validation/PearsonCorrelationMatrix.py

@@ -88,4 +88,4 @@ def PearsonCorrelationMatrix(dataset):
 
     fig = go.Figure(data=[heatmap], layout=layout)
 
-    return fig, RawData(correlation_matrix=corr_matrix)
+    return fig, RawData(correlation_matrix=corr_matrix, dataset=dataset.input_id)

validmind/tests/data_validation/ProtectedClassesCombination.py

@@ -206,5 +206,7 @@ def ProtectedClassesCombination(dataset, model, protected_classes=None):
             metrics_frame=mf,
             demographic_parity_ratios=m_dpr,
             equalized_odds_ratios=m_eqo,
+            model=model.input_id,
+            dataset=dataset.input_id,
         ),
     )

validmind/tests/data_validation/ProtectedClassesDescription.py

@@ -6,7 +6,7 @@
 import pandas as pd
 import plotly.graph_objects as go
 
-from validmind import tags, tasks
+from validmind import RawData, tags, tasks
 from validmind.logging import get_logger
 
 logger = get_logger(__name__)
@@ -127,4 +127,4 @@ def ProtectedClassesDescription(dataset, protected_classes=None):
         ["Protected Class", "Count"], ascending=[True, False]
     )
 
-    return (stats_df, *figures)
+    return (stats_df, *figures, RawData(dataset=dataset.input_id))