validmind 2.7.9__py3-none-any.whl → 2.8.10__py3-none-any.whl

validmind/__init__.py

@@ -30,8 +30,12 @@ vm.init(
 
 After you have pasted the code snippet into your development source code and executed the code, the Python Library API will register with ValidMind. You can now use the ValidMind Library to document and test your models, and to upload to the ValidMind Platform.
 """
+import threading
 import warnings
 
+import pkg_resources
+from IPython.display import HTML, display
+
 # Ignore Numba warnings. We are not requiring this package directly
 from numba.core.errors import NumbaDeprecationWarning, NumbaPendingDeprecationWarning
 
@@ -51,30 +55,74 @@ from .client import (  # noqa: E402
 )
 from .tests.decorator import tags, tasks, test
 from .tests.run import print_env
+from .utils import is_notebook, parse_version
 from .vm_models.result import RawData
 
+__shown = False
+
+
+def show_warning(installed, running):
+    global __shown
+
+    if __shown:
+        return
+    __shown = True
+
+    message = (
+        f"⚠️ This kernel is running an older version of validmind ({running}) "
+        f"than the latest version installed on your system ({installed}).\n\n"
+        "You may need to restart the kernel if you are experiencing issues."
+    )
+    display(HTML(f"<div style='color: red;'>{message}</div>"))
+
+
+def check_version():
+    # get the installed vs running version of validmind
+    # to make sure we are using the latest installed version
+    # in case user has updated the package but forgot to restart the kernel
+    installed = pkg_resources.get_distribution("validmind").version
+    running = __version__
+
+    if parse_version(installed) > parse_version(running):
+        show_warning(installed, running)
+
+    # Schedule the next check for 5 minutes from now
+    timer = threading.Timer(300, check_version)
+    timer.daemon = True
+    timer.start()
+
+
+if is_notebook():
+    check_version()
+
 __all__ = [  # noqa
     "__version__",
-    # Python Library API
-    "datasets",
-    "errors",
-    "get_test_suite",
+    # main library API
     "init",
+    "reload",
     "init_dataset",
     "init_model",
     "init_r_model",
     "preview_template",
-    "print_env",
-    "RawData",
-    "reload",
     "run_documentation_tests",
+    # log metric function (for direct/bulk/retroactive logging of metrics)
+    "log_metric",
+    # test suite functions (less common)
+    "get_test_suite",
     "run_test_suite",
+    # helper functions (for troubleshooting)
+    "print_env",
+    # decorators (for building tests
     "tags",
     "tasks",
     "test",
-    "tests",
-    "test_suites",
+    # raw data (for post-processing test results and building tests)
+    "RawData",
+    # submodules
+    "datasets",
+    "errors",
     "vm_models",
+    "tests",
     "unit_metrics",
-    "log_metric",
+    "test_suites",
 ]

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.7.9"
+__version__ = "2.8.10"

validmind/ai/test_descriptions.py

@@ -4,70 +4,24 @@
 
 import json
 import os
-import re
 from concurrent.futures import ThreadPoolExecutor
 from typing import List, Optional, Union
 
 import tiktoken
-from jinja2 import Template
 
 from ..client_config import client_config
 from ..logging import get_logger
 from ..utils import NumpyEncoder, md_to_html, test_id_to_name
 from ..vm_models.figure import Figure
 from ..vm_models.result import ResultTable
-from .utils import DescriptionFuture, get_client_and_model
+from .utils import DescriptionFuture
 
 __executor = ThreadPoolExecutor()
-__prompt = None
 
 logger = get_logger(__name__)
 
 
-def _load_prompt():
-    global __prompt
-
-    if not __prompt:
-        folder_path = os.path.join(os.path.dirname(__file__), "test_result_description")
-        with open(os.path.join(folder_path, "system.jinja"), "r") as f:
-            system_prompt = f.read()
-        with open(os.path.join(folder_path, "user.jinja"), "r") as f:
-            user_prompt = f.read()
-
-        __prompt = (Template(system_prompt), Template(user_prompt))
-
-    return __prompt
-
-
-def prompt_to_message(role, prompt):
-    if "[[IMAGE:" not in prompt:
-        return {"role": role, "content": prompt}
-
-    content = []
-
-    # Regex pattern to find [[IMAGE:<b64-data>]] markers
-    pattern = re.compile(r"\[\[IMAGE:(.*?)\]\]", re.DOTALL)
-
-    last_index = 0
-    for match in pattern.finditer(prompt):
-        # Text before the image marker
-        start, end = match.span()
-        if start > last_index:
-            content.append({"type": "text", "text": prompt[last_index:start]})
-
-        content.append({"type": "image_url", "image_url": {"url": match.group(1)}})
-
-        last_index = end
-
-    # Text after the last image
-    if last_index < len(prompt):
-        content.append({"type": "text", "text": prompt[last_index:]})
-
-    return {"role": role, "content": content}
-
-
 def _get_llm_global_context():
-
     # Get the context from the environment variable
     context = os.getenv("VALIDMIND_LLM_DESCRIPTIONS_CONTEXT", "")
 
@@ -115,13 +69,13 @@ def generate_description(
     title: Optional[str] = None,
 ):
     """Generate the description for the test results"""
+    from validmind.api_client import generate_test_result_description
+
     if not tables and not figures and not metric:
         raise ValueError(
             "No tables, unit metric or figures provided - cannot generate description"
         )
 
-    client, model = get_client_and_model()
-
     # get last part of test id
     test_name = title or test_id.split(".")[-1]
 
@@ -145,29 +99,18 @@ def generate_description(
     else:
         summary = None
 
-    context = _get_llm_global_context()
-
-    input_data = {
-        "test_name": test_name,
-        "test_description": test_description,
-        "title": title,
-        "summary": _truncate_summary(summary, test_id),
-        "figures": [figure._get_b64_url() for figure in ([] if tables else figures)],
-        "context": context,
-    }
-    system, user = _load_prompt()
-
-    messages = [
-        prompt_to_message("system", system.render(input_data)),
-        prompt_to_message("user", user.render(input_data)),
-    ]
-    response = client.chat.completions.create(
-        model=model,
-        temperature=0.0,
-        messages=messages,
-    )
-
-    return response.choices[0].message.content
+    return generate_test_result_description(
+        {
+            "test_name": test_name,
+            "test_description": test_description,
+            "title": title,
+            "summary": _truncate_summary(summary, test_id),
+            "figures": [
+                figure._get_b64_url() for figure in ([] if tables else figures)
+            ],
+            "context": _get_llm_global_context(),
+        }
+    )["content"]
 
 
 def background_generate_description(
@@ -238,7 +181,8 @@ def get_result_description(
     # Check the feature flag first, then the environment variable
     llm_descriptions_enabled = (
         client_config.can_generate_llm_test_descriptions()
-        and os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1") not in ["0", "false"]
+        and os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1").lower()
+        not in ["0", "false"]
     )
 
     # TODO: fix circular import

validmind/api_client.py

@@ -194,6 +194,7 @@ def init(
     api_host: Optional[str] = None,
     model: Optional[str] = None,
     monitoring: bool = False,
+    generate_descriptions: Optional[bool] = None,
 ):
     """
     Initializes the API client instances and calls the /ping endpoint to ensure
@@ -209,7 +210,7 @@ def init(
         api_secret (str, optional): The API secret. Defaults to None.
         api_host (str, optional): The API host. Defaults to None.
         monitoring (bool): The ongoing monitoring flag. Defaults to False.
-
+        generate_descriptions (bool): Whether to use GenAI to generate test result descriptions. Defaults to True.
     Raises:
         ValueError: If the API key and secret are not provided
     """
@@ -235,6 +236,9 @@ def init(
 
     _monitoring = monitoring
 
+    if generate_descriptions is not None:
+        os.environ["VALIDMIND_LLM_DESCRIPTIONS_ENABLED"] = str(generate_descriptions)
+
     reload()
 
 
@@ -487,3 +491,16 @@ def get_ai_key() -> Dict[str, Any]:
         raise_api_error(r.text)
 
     return r.json()
+
+
+def generate_test_result_description(test_result_data: Dict[str, Any]) -> str:
+    r = requests.post(
+        url=_get_url("ai/generate/test_result_description"),
+        headers=_get_api_headers(),
+        json=test_result_data,
+    )
+
+    if r.status_code != 200:
+        raise_api_error(r.text)
+
+    return r.json()

validmind/errors.py

@@ -228,7 +228,7 @@ class MissingRExtrasError(BaseError):
     def description(self, *args, **kwargs):
         return (
             self.message
-            or "ValidMind r-support needs to be installed: `pip install validmind[r-support]`"
+            or "`rpy2` is required to use R models. Please install it with `pip install rpy2`"
         )
 
 

validmind/models/r_model.py

@@ -5,6 +5,7 @@
 import numpy as np
 import pandas as pd
 
+from validmind.errors import MissingRExtrasError
 from validmind.logging import get_logger
 from validmind.vm_models.model import VMModel
 
@@ -125,7 +126,10 @@ class RModel(VMModel):
         """
         Converts the predicted probabilities to classes
         """
-        from rpy2.robjects import pandas2ri
+        try:
+            from rpy2.robjects import pandas2ri
+        except ImportError:
+            raise MissingRExtrasError()
 
         # Activate the pandas conversion for rpy2
         pandas2ri.activate()

validmind/tests/comparison.py

@@ -15,7 +15,7 @@ from validmind.vm_models.figure import (
     is_png_image,
 )
 from validmind.vm_models.input import VMInput
-from validmind.vm_models.result import ResultTable, TestResult
+from validmind.vm_models.result import RawData, ResultTable, TestResult
 
 logger = get_logger(__name__)
 
@@ -312,6 +312,25 @@ def get_comparison_test_configs(
     return test_configs
 
 
+def _combine_raw_data(results: List[TestResult]) -> RawData:
+    """Combine RawData objects"""
+    attribute_names = results[0].raw_data.__dict__.keys()
+
+    # check that all the raw data objects have the same attributes
+    for result in results:
+        if not isinstance(result.raw_data, RawData):
+            raise ValueError("All raw data objects must be of type RawData")
+        if result.raw_data.__dict__.keys() != attribute_names:
+            raise ValueError("RawData objects must have the same attributes")
+
+    return RawData(
+        **{
+            key: [getattr(result.raw_data, key) for result in results]
+            for key in attribute_names
+        }
+    )
+
+
 def combine_results(
     results: List[TestResult],
 ) -> Tuple[List[Any], Dict[str, List[Any]], Dict[str, List[Any]]]:
@@ -338,6 +357,9 @@ def combine_results(
     # handle threshold tests (i.e. tests that have pass/fail bool status)
     if results[0].passed is not None:
         combined_outputs.append(all(result.passed for result in results))
+    # handle raw data (if any)
+    if results[0].raw_data:
+        combined_outputs.append(_combine_raw_data(results))
 
     # combine inputs and params
     combined_inputs = {}
@@ -359,4 +381,8 @@ def combine_results(
     combined_inputs = _combine_dict_values(combined_inputs)
     combined_params = _combine_dict_values(combined_params)
 
-    return combined_outputs, combined_inputs, combined_params
+    return (
+        tuple(combined_outputs),
+        combined_inputs,
+        combined_params,
+    )

validmind/tests/data_validation/FeatureTargetCorrelationPlot.py

@@ -52,16 +52,10 @@ def FeatureTargetCorrelationPlot(dataset, fig_height=600):
     - Not apt for models that employ complex feature interactions, like Decision Trees or Neural Networks, as the test
     may not accurately reflect their importance.
     """
-
-    # Filter DataFrame based on features and target_column
     df = dataset.df[dataset.feature_columns + [dataset.target_column]]
 
-    fig = _visualize_feature_target_correlation(df, dataset.target_column, fig_height)
-
-    correlations = (
-        df.corr(numeric_only=True)[dataset.target_column]
-        .drop(dataset.target_column)
-        .to_frame()
+    fig, correlations = _visualize_feature_target_correlation(
+        df, dataset.target_column, fig_height
     )
 
     return fig, RawData(correlation_data=correlations)
@@ -100,4 +94,5 @@ def _visualize_feature_target_correlation(df, target_column, fig_height):
         yaxis_title="",
         height=fig_height,  # Adjust the height value as needed
     )
-    return fig
+
+    return fig, correlations

validmind/tests/model_validation/ContextualRecall.py

@@ -118,4 +118,4 @@ def ContextualRecall(dataset, model):
     # Create a DataFrame from all collected statistics
     result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
 
-    return (result_df, *tuple(figures), RawData(contextual_recall_scores=metrics_df))
+    return (result_df, *figures, RawData(contextual_recall_scores=metrics_df))

validmind/tests/model_validation/MeteorScore.py

@@ -117,4 +117,4 @@ def MeteorScore(dataset, model):
     # Create a DataFrame from all collected statistics
     result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
 
-    return (result_df, *tuple(figures), RawData(meteor_scores=metrics_df))
+    return (result_df, *figures, RawData(meteor_scores=metrics_df))

validmind/tests/model_validation/ToxicityScore.py

@@ -141,7 +141,7 @@ def ToxicityScore(dataset, model):
 
     return (
         result_df,
-        *tuple(figures),
+        *figures,
         RawData(
             input_toxicity_df=input_df,
             true_toxicity_df=true_df,

validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py

@@ -151,4 +151,4 @@ def StabilityAnalysisRandomNoise(
         mean_similarity_threshold,
     )
 
-    return result, RawData(original_perturbed_similarity=raw_data)
+    return *result, RawData(original_perturbed_similarity=raw_data)

validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py

@@ -107,4 +107,4 @@ def StabilityAnalysisSynonyms(
         mean_similarity_threshold,
     )
 
-    return result, RawData(original_perturbed_similarity=raw_data)
+    return *result, RawData(original_perturbed_similarity=raw_data)

validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py

@@ -134,4 +134,4 @@ def StabilityAnalysisTranslation(
         mean_similarity_threshold,
     )
 
-    return result, RawData(original_perturbed_similarity=raw_data)
+    return *result, RawData(original_perturbed_similarity=raw_data)

validmind/tests/model_validation/embeddings/utils.py

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import numpy as np
-import pandas as pd
 import plotly.express as px
 from sklearn.metrics.pairwise import cosine_similarity
 
@@ -18,14 +17,12 @@ def create_stability_analysis_result(
         original_embeddings, perturbed_embeddings
     ).diagonal()
 
-    # create a raw dataframe of the original, perturbed and similarity
-    raw_data = pd.DataFrame(
-        {
-            "original": original_embeddings,
-            "perturbed": perturbed_embeddings,
-            "similarity": similarities,
-        }
-    )
+    # Store raw data in a dictionary
+    raw_data = {
+        "original_embeddings": original_embeddings,
+        "perturbed_embeddings": perturbed_embeddings,
+        "similarities": similarities,
+    }
 
     mean = np.mean(similarities)
     passed = mean > mean_similarity_threshold

validmind/tests/model_validation/ragas/utils.py

@@ -4,24 +4,25 @@
 
 import os
 
-from validmind.ai.utils import get_client_and_model
-from validmind.client_config import client_config
+from validmind.ai.utils import get_client_and_model, is_configured
 
 EMBEDDINGS_MODEL = "text-embedding-3-small"
 
 
 def get_ragas_config():
-    if not client_config.can_generate_llm_test_descriptions():
-        raise ValueError(
-            "LLM based descriptions are not enabled in the current configuration."
-        )
-
     # import here since its an optional dependency
     try:
         from langchain_openai import ChatOpenAI, OpenAIEmbeddings
     except ImportError:
         raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
 
+    if not is_configured():
+        raise ValueError(
+            "LLM is not configured. Please set an `OPENAI_API_KEY` environment variable "
+            "or ensure that you are connected to the ValidMind API and ValidMind AI is "
+            "enabled for your account."
+        )
+
     client, model = get_client_and_model()
     os.environ["OPENAI_API_BASE"] = str(client.base_url)
 

validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py

@@ -62,18 +62,14 @@ def MinimumROCAUCScore(dataset: VMDataset, model: VMModel, min_threshold: float
         lb = LabelBinarizer()
         lb.fit(y_true)
 
-        y_true_binarized = lb.transform(y_true)
-        y_score_binarized = lb.transform(dataset.y_pred(model))
-
         roc_auc = roc_auc_score(
-            y_true=y_true_binarized,
-            y_score=y_score_binarized,
+            y_true=lb.transform(y_true),
+            y_score=lb.transform(dataset.y_pred(model)),
             average="macro",
         )
 
     else:
-        y_score_prob = dataset.y_prob(model)
-        roc_auc = roc_auc_score(y_true=y_true, y_score=y_score_prob)
+        roc_auc = roc_auc_score(y_true=y_true, y_score=dataset.y_prob(model))
 
     return [
         {

validmind/tests/model_validation/statsmodels/RegressionModelSummary.py

@@ -45,17 +45,17 @@ def RegressionModelSummary(dataset: VMDataset, model: VMModel):
     - A high R-Squared or Adjusted R-Squared may not necessarily indicate a good model, especially in cases of
     overfitting.
     """
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
     return [
         {
             "Independent Variables": dataset.feature_columns,
-            "R-Squared": r2_score(dataset.y, dataset.y_pred(model)),
+            "R-Squared": r2_score(y_true, y_pred),
             "Adjusted R-Squared": adj_r2_score(
-                dataset.y,
-                dataset.y_pred(model),
-                len(dataset.y),
-                len(dataset.feature_columns),
+                y_true, y_pred, len(y_true), len(dataset.feature_columns)
             ),
-            "MSE": mean_squared_error(dataset.y, dataset.y_pred(model), squared=True),
-            "RMSE": mean_squared_error(dataset.y, dataset.y_pred(model), squared=False),
+            "MSE": mean_squared_error(y_true, y_pred, squared=True),
+            "RMSE": mean_squared_error(y_true, y_pred, squared=False),
         }
     ]

validmind/tests/prompt_validation/ai_powered_test.py

@@ -4,8 +4,7 @@
 
 import re
 
-from validmind.ai.utils import get_client_and_model
-from validmind.client_config import client_config
+from validmind.ai.utils import get_client_and_model, is_configured
 
 missing_prompt_message = """
 Cannot run prompt validation tests on a model with no prompt.
@@ -25,9 +24,11 @@ def call_model(
     system_prompt: str, user_prompt: str, temperature: float = 0.0, seed: int = 42
 ):
     """Call LLM with the given prompts and return the response"""
-    if not client_config.can_generate_llm_test_descriptions():
+    if not is_configured():
         raise ValueError(
-            "LLM based descriptions are not enabled for your organization."
+            "LLM is not configured. Please set an `OPENAI_API_KEY` environment variable "
+            "or ensure that you are connected to the ValidMind API and ValidMind AI is "
+            "enabled for your account."
         )
 
     client, model = get_client_and_model()

validmind/tests/run.py

@@ -256,7 +256,7 @@ def _run_comparison_test(
     combined_outputs, combined_inputs, combined_params = combine_results(results)
 
     return build_test_result(
-        outputs=tuple(combined_outputs),
+        outputs=combined_outputs,
         test_id=test_id,
         test_doc=test_doc,
         inputs=combined_inputs,
@@ -400,5 +400,9 @@ def run_test(  # noqa: C901
 
 
 def print_env():
+    """Prints a log of the running environment for debugging.
+
+    Output includes: ValidMind Library version, operating system details, installed dependencies, and the ISO 8601 timestamp at log creation.
+    """
     e = _get_run_metadata()
     pprint.pp(e)

validmind/utils.py

@@ -60,6 +60,19 @@ pylab.rcParams.update(params)
 logger = get_logger(__name__)
 
 
+def parse_version(version: str) -> tuple[int, ...]:
+    """
+    Parse a semver version string into a tuple of major, minor, patch integers
+
+    Args:
+        version (str): The semantic version string to parse
+
+    Returns:
+        tuple[int, ...]: A tuple of major, minor, patch integers
+    """
+    return tuple(int(x) for x in version.split(".")[:3])
+
+
 def is_notebook() -> bool:
     """
     Checks if the code is running in a Jupyter notebook or IPython shell
@@ -110,6 +123,7 @@ class NumpyEncoder(json.JSONEncoder):
             self.is_numpy_ndarray: lambda obj: obj.tolist(),
             self.is_numpy_bool: lambda obj: bool(obj),
             self.is_pandas_timestamp: lambda obj: str(obj),
+            self.is_numpy_datetime64: lambda obj: str(obj),
             self.is_set: lambda obj: list(obj),
             self.is_quantlib_date: lambda obj: obj.ISO(),
             self.is_generic_object: self.handle_generic_object,
@@ -142,6 +156,9 @@ class NumpyEncoder(json.JSONEncoder):
     def is_pandas_timestamp(self, obj):
         return isinstance(obj, pd.Timestamp)
 
+    def is_numpy_datetime64(self, obj):
+        return isinstance(obj, np.datetime64)
+
     def is_set(self, obj):
         return isinstance(obj, set)
 
@@ -152,11 +169,12 @@ class NumpyEncoder(json.JSONEncoder):
         return isinstance(obj, object)
 
     def handle_generic_object(self, obj):
-        return (
-            obj.__str__()
-            if type(obj).__dict__.get("__str__")
-            else str(obj).split(".")[1].split(" ")[0]
-        )
+        try:
+            if hasattr(obj, "__str__"):
+                return obj.__str__()
+            return obj.__class__.__name__
+        except Exception:
+            return str(type(obj).__name__)
 
     def encode(self, obj):
         obj = nan_to_none(obj)
@@ -177,6 +195,18 @@ class HumanReadableEncoder(NumpyEncoder):
             else obj.tolist()
         )
 
+    def default(self, obj):
+        if self.is_dataframe(obj):
+            return {
+                "type": str(type(obj)),
+                "preview": obj.head(5).to_dict(orient="list"),
+                "shape": f"{obj.shape[0]} rows x {obj.shape[1]} columns",
+            }
+        return super().default(obj)
+
+    def is_dataframe(self, obj):
+        return isinstance(obj, pd.DataFrame)
+
 
 def get_full_typename(o: Any) -> Any:
     """We determine types based on type names so we don't have to import

validmind/vm_models/result/result.py

@@ -171,6 +171,7 @@ class TestResult(Result):
     metadata: Optional[Dict[str, Any]] = None
     _was_description_generated: bool = False
     _unsafe: bool = False
+    _client_config_cache: Optional[Any] = None
 
     def __post_init__(self):
         if self.ref_id is None:
@@ -329,13 +330,50 @@ class TestResult(Result):
 
         return VBox(widgets)
 
+    @classmethod
+    def _get_client_config(cls):
+        """Get the client config, loading it if not cached"""
+        if cls._client_config_cache is None:
+            api_client.reload()
+            cls._client_config_cache = api_client.client_config
+
+            if cls._client_config_cache is None:
+                raise ValueError(
+                    "Failed to load client config: api_client.client_config is None"
+                )
+
+            if not hasattr(cls._client_config_cache, "documentation_template"):
+                raise ValueError(
+                    "Invalid client config: missing documentation_template"
+                )
+
+        return cls._client_config_cache
+
+    def check_result_id_exist(self):
+        """Check if the result_id exists in any test block across all sections"""
+        client_config = self._get_client_config()
+
+        # Iterate through all sections
+        for section in client_config.documentation_template["sections"]:
+            blocks = section.get("contents", [])
+            # Check each block in the section
+            for block in blocks:
+                if (
+                    block.get("content_type") == "test"
+                    and block.get("content_id") == self.result_id
+                ):
+                    return
+
+        logger.info(
+            f"Test driven block with result_id {self.result_id} does not exist in model's document"
+        )
+
     def _validate_section_id_for_block(
         self, section_id: str, position: Union[int, None] = None
     ):
         """Validate the section_id exits on the template before logging"""
-        api_client.reload()
+        client_config = self._get_client_config()
         found = False
-        client_config = api_client.client_config
 
         for section in client_config.documentation_template["sections"]:
             if section["id"] == section_id:
@@ -440,6 +478,9 @@ class TestResult(Result):
             unsafe (bool): If True, log the result even if it contains sensitive data
                 i.e. raw data from input datasets
         """
+
+        self.check_result_id_exist()
+
         if not unsafe:
             for table in self.tables or []:
                 check_for_sensitive_data(table.data, self._get_flat_inputs())

{validmind-2.7.9.dist-info → validmind-2.8.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: validmind
-Version: 2.7.9
+Version: 2.8.10
 Summary: ValidMind Library
 License: Commercial License
 Author: Andres Rodriguez
@@ -15,8 +15,8 @@ Provides-Extra: all
 Provides-Extra: huggingface
 Provides-Extra: llm
 Provides-Extra: pytorch
-Provides-Extra: r-support
 Requires-Dist: aiohttp[speedups]
+Requires-Dist: anywidget (>=0.9.13,<0.10.0)
 Requires-Dist: arch
 Requires-Dist: bert-score (>=0.3.13)
 Requires-Dist: catboost
@@ -42,7 +42,6 @@ Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"
 Requires-Dist: python-dotenv
 Requires-Dist: ragas (>=0.2.3) ; extra == "all" or extra == "llm"
 Requires-Dist: rouge (>=1)
-Requires-Dist: rpy2 (>=3.5.10,<4.0.0) ; extra == "all" or extra == "r-support"
 Requires-Dist: scikit-learn (<1.6.0)
 Requires-Dist: scipy
 Requires-Dist: scorecardpy (>=0.1.9.6,<0.2.0.0)
@@ -53,6 +52,7 @@ Requires-Dist: shap (==0.44.1)
 Requires-Dist: statsmodels
 Requires-Dist: tabulate (>=0.8.9,<0.9.0)
 Requires-Dist: textblob (>=0.18.0.post0,<0.19.0)
+Requires-Dist: tiktoken
 Requires-Dist: torch (>=1.10.0) ; extra == "all" or extra == "llm" or extra == "pytorch"
 Requires-Dist: tqdm
 Requires-Dist: transformers (>=4.32.0,<5.0.0) ; extra == "all" or extra == "huggingface" or extra == "llm"
@@ -131,6 +131,7 @@ The ValidMind Library has optional dependencies that can be installed separately
 - **R Models**: To use R models with the ValidMind Library, install the `r` extra:
 
     ```bash
-    pip install validmind[r-support]
+    pip install validmind
+    pip install rpy2
     ```
 

{validmind-2.7.9.dist-info → validmind-2.8.10.dist-info}/RECORD

@@ -1,13 +1,8 @@
-validmind/__init__.py,sha256=7qIV3pfZ9K4Gnq11Nkm8TutNKdAmPtUY9DZ0N_OaKks,2738
-validmind/__version__.py,sha256=_43Rb_ixjzQz00TkrUhA8ua5QXQmT2qU8OCNFLRevrs,22
-validmind/ai/test_descriptions.py,sha256=ZApQ9PGBP1PI6HgyRG7Dwfo1C8t_89WRhVv1-H-i0Y4,8438
-validmind/ai/test_result_description/config.yaml,sha256=E1gPd-uv-MzdrWZA_rP6LSk8pVmkYijx6v78hZ8ceL0,787
-validmind/ai/test_result_description/context.py,sha256=ebKulFMpXTDLqd6lOHAsG200GmLNnhnu7sMDnbo2Dhc,2339
-validmind/ai/test_result_description/image_processing.py,sha256=JNaO1zyM9293WWuyzUp1meQQbHuut0XN4kKUGzQTwYY,4061
-validmind/ai/test_result_description/system.jinja,sha256=BjMvZCC3UXEH8p3VPpnHtGjhnqnbNcEG2_kYZ_QZrgg,2358
-validmind/ai/test_result_description/user.jinja,sha256=CmqPQQiqdXjxtq47wFCZ-IT5csliWsRVM04psKxzXc4,689
+validmind/__init__.py,sha256=4ukEysjRwf6X2RcK8_OVY-z5fTjnM4GQIiVERFyzRPY,4194
+validmind/__version__.py,sha256=93aDjSnP93wggEKIISA3t5wll8Pc68HgFa7UDIxqdwo,23
+validmind/ai/test_descriptions.py,sha256=VQ2LaWXvSEQZ8569TN9DYDfgDLlMv96wIjUb9MGJlHk,6882
 validmind/ai/utils.py,sha256=YHqXtmACjcL5imDS9_nzmz8MhQJzK3VybcDXMbj1SbQ,4168
-validmind/api_client.py,sha256=fo2NbgdW5y0sUmpSLLRJ7pGs4eU8LvnjC3l-eZzSTSo,14842
+validmind/api_client.py,sha256=II9dggnuZwagZQaTHMqCkfS9fKrMCZO0cF6at0kBO2w,15456
 validmind/client.py,sha256=lOv4lSZGDOUMxOa2FpNgAiT_GaEolffZTfvljewhl2I,18595
 validmind/client_config.py,sha256=CzbeS9GZrgAdx-6DssRK5XwEMOcujQVRgji6EtAzxYI,1358
 validmind/datasets/__init__.py,sha256=c0hQZN_6GrUEJxdFHdQaEsQrSYNABG84ZCY0H-PzOZk,260
@@ -64,7 +59,7 @@ validmind/datasets/regression/models/fred_loan_rates_model_2.pkl,sha256=J1ukMdeF
 validmind/datasets/regression/models/fred_loan_rates_model_3.pkl,sha256=IogZPcUQc1F_v11fR6KWT-nRt5JzvK5f7p4Hrw7vLps,40063
 validmind/datasets/regression/models/fred_loan_rates_model_4.pkl,sha256=cSxhpcrI4hCbxCwZwE2-nr7KObbWpDii3NzpECoXmmM,48292
 validmind/datasets/regression/models/fred_loan_rates_model_5.pkl,sha256=FkNLHq9xkPMbYks_vyMjFL371mw9SQYbP1iX9lY4Ljo,60343
-validmind/errors.py,sha256=QrobGZtTH7SC91pFHj2Yal11gVxX0np9W2fKBfVxn-E,8074
+validmind/errors.py,sha256=_zM-CABfIGz6hirW4nuukyOgYkFCZ8xh55Jn5DSjd6g,8074
 validmind/html_templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 validmind/html_templates/content_blocks.py,sha256=vFMRS4Ogq4RZq88WzG3teNEOq3U4OLgLDzD3lBx4h-g,4050
 validmind/input_registry.py,sha256=8C_mrhgLT72hwbt_lo3ZwXb5NCyIcSuCQI1HdJ3bK2A,1042
@@ -76,7 +71,7 @@ validmind/models/huggingface.py,sha256=DMHekLpWi6c4N0svh-3G0NYYwzxPXOvqiU95M4Qff
 validmind/models/metadata.py,sha256=PMcdYuACkSPvuG8io5BhZeMwclQr_q79mXbvd1SC-7I,1665
 validmind/models/pipeline.py,sha256=nSskKWxaS4SGmx_B0IAvS5ogDZyh6tdx_aUkyxSXt88,2051
 validmind/models/pytorch.py,sha256=aAEUWtISwLh-PMvHkcLwBEbBStAByt4J-NpK-Ndv38E,1826
-validmind/models/r_model.py,sha256=eYdpCREgBpYv-PxJDuG91I77OOAx3-43FoaYT560ziE,7172
+validmind/models/r_model.py,sha256=TPUwPmxz3cNzJ1bAA5vz6P9xS6deVcLTuIO1e7rD1vY,7306
 validmind/models/sklearn.py,sha256=lOCJlP2wvd5IJHtBS1XG9FXrtIvO_f8xm2Qp1UdsiBw,2406
 validmind/template.py,sha256=-j7UmM9v7I_VIZltWrmX5scbeDTbRDrR7hTQUzy6AVg,7307
 validmind/test_suites/__init__.py,sha256=wC_ZgVykFZAOrs1LTM7bE8r7mTSjxF54F9amUdT3nT4,6953
@@ -95,7 +90,7 @@ validmind/test_suites/time_series.py,sha256=3hzWc9gXHBf8aMecD-1YYGFS5RI1o8A-eor9
 validmind/tests/__init__.py,sha256=U6wUS7R8lYFjwUZmAkG7gC8Av9Z4TTDZS7uWZqbzxVM,1665
 validmind/tests/__types__.py,sha256=3if3CVI-YHWQpX0t_8WYh1a6dd01us0oP7r2ZsX2NX0,12589
 validmind/tests/_store.py,sha256=p1qBYOkdphd1aOWwnL6ybPfz3qSWJs2engLv-zFVl5A,2368
-validmind/tests/comparison.py,sha256=lqpaRXZVNqMYNf_mbtVXOGkEtygL4edAyvQDwX4nFf8,12488
+validmind/tests/comparison.py,sha256=7LJFr_6sVVOAxLRX1LYylJ8oZF0UdrVV7tFfWIUfcWI,13336
 validmind/tests/data_validation/ACFandPACFPlot.py,sha256=Xo_xildBWlKJSbycH-IqXh5PkpBtEFym5WN_m9VKDwU,4276
 validmind/tests/data_validation/ADF.py,sha256=UHIYCJuqQYJTtOvdzj6ALJcLH8kCprr0PlfESIx_G_E,3969
 validmind/tests/data_validation/AutoAR.py,sha256=QQGezpGkYS0m0isT3BDhBAGMh8CS8ZU2HsU3G4p9lY0,5238
@@ -111,7 +106,7 @@ validmind/tests/data_validation/DescriptiveStatistics.py,sha256=l6Ne4y5A9HC2vlPs
 validmind/tests/data_validation/DickeyFullerGLS.py,sha256=8zR7I77082RfYOyOlmYX7OLURXJ8_G1Yh9Esb27FzFQ,4193
 validmind/tests/data_validation/Duplicates.py,sha256=HAEHRFwFZovJU-wBWea0KJREsJCd9R8jarazxJ3ZqI8,3219
 validmind/tests/data_validation/EngleGrangerCoint.py,sha256=kNBbxLYweF8qTF5JVRzcyXq3aKLhkN_1iv3mwwskTBU,4503
-validmind/tests/data_validation/FeatureTargetCorrelationPlot.py,sha256=6q-RYIC9hIk0mcKuIntpikYsO3UguCHIeeM8wXI1FAI,4441
+validmind/tests/data_validation/FeatureTargetCorrelationPlot.py,sha256=IEQYOakooUJRLe1CghVk0u4RJdLZ3IA-nnYveqgKJVE,4281
 validmind/tests/data_validation/HighCardinality.py,sha256=Li6xa46crnmdx9XliuHbDDVH4_KzcUT4T7Z16cRYQnY,3546
 validmind/tests/data_validation/HighPearsonCorrelation.py,sha256=f8AtgstLTY5msFfDkCEe3hScHmVr8A2rL4OgdJSdkYQ,3779
 validmind/tests/data_validation/IQROutliersBarPlot.py,sha256=BrhqjiV5EpQAgPI_6GUdF5NLA3O5GbJk50dXbi-9G0U,5182
@@ -172,9 +167,9 @@ validmind/tests/load.py,sha256=UpMHEMolnwQ8z3EYNJvNj186H3JUgQvz_eRCBrngtBo,11003
 validmind/tests/model_validation/BertScore.py,sha256=R6jnQ9cDKbHpaJFtf2wTHkWtKX_DCRzNm6NkOmgGLJ8,5757
 validmind/tests/model_validation/BleuScore.py,sha256=xFPkmxVbM4ST3Bvi5IH9Sh_BJU_Civ-Slh66AoZgRec,5134
 validmind/tests/model_validation/ClusterSizeDistribution.py,sha256=mZcLjbHjGJ4ltt5juyuGRC5Fm7oqDB6ZPBxCvd1-h_k,3284
-validmind/tests/model_validation/ContextualRecall.py,sha256=maDskSjpMFDe7wbQIMS5TK_fG04I9QY3oM63kRgEhhE,5263
+validmind/tests/model_validation/ContextualRecall.py,sha256=0W1xkiDff09ql7x9N0JW4qafCKyLdCttbbybDTJE9pQ,5256
 validmind/tests/model_validation/FeaturesAUC.py,sha256=LWURhngE04h3r_UA0JQIyoHMYsrJTRQHHlmK2UnR0P4,4010
-validmind/tests/model_validation/MeteorScore.py,sha256=0AhyCHy4eYJvWduq9JY0RRtbv4rz19OcsEWCdTdHhmQ,5253
+validmind/tests/model_validation/MeteorScore.py,sha256=PXLfyZ1tFDoYfZLniAxOpfSRJDFvGdaUm5LKH_5XotA,5246
 validmind/tests/model_validation/ModelMetadata.py,sha256=UiDANMTqAy0DURnnTzImYNS-3Z8sE4yFjg1c2S5YFS8,2521
 validmind/tests/model_validation/ModelPredictionResiduals.py,sha256=mZmE0RWTUHigJJ3EvrzWcHjb5nvePYGH2PvZTQl2ZNc,3524
 validmind/tests/model_validation/RegardScore.py,sha256=cE3BmI1SQ3rlRbrREio8mooaG2BQ0BC_AHMFexbJ3Xk,5562
@@ -184,7 +179,7 @@ validmind/tests/model_validation/TimeSeriesPredictionWithCI.py,sha256=JanMDaf-76
 validmind/tests/model_validation/TimeSeriesPredictionsPlot.py,sha256=zoxONIl_a_mfOLuxnfVEMNQ3jL72pp5J4BDAQP5e-P0,2280
 validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py,sha256=JIyER-vwXnQyHdxmpCULYm4iMkB_LaLHHQuzyygumbw,4000
 validmind/tests/model_validation/TokenDisparity.py,sha256=_gn-LSghJa8hBtCKxfNKw7rAC8h1uyrFhcX8BIaPoUs,4422
-validmind/tests/model_validation/ToxicityScore.py,sha256=0XjxALRwXh_gM-UwCXpAopgE-ZsgNQOlxbWOgef01Kw,5472
+validmind/tests/model_validation/ToxicityScore.py,sha256=s-EPOn6Apo7dD_bw0wcmTez3P-lv6hfLV61UHuLMsgg,5465
 validmind/tests/model_validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 validmind/tests/model_validation/embeddings/ClusterDistribution.py,sha256=q0v88QINg5BQWuFbQrvVb_lP-lLEQt_h1f3Af4TX6Bc,3056
 validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py,sha256=Xtycolawy2Z7hhShfdlLpYvpc1lgQrISCL75ltr8s1g,5138
@@ -196,11 +191,11 @@ validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py,sha25
 validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py,sha256=TvB2yi5C1OXtCgiXy4z9CBGbUiNEwEVzgG69gmpb3hQ,3393
 validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py,sha256=lF4QC5Ueihxt-ON1GyUTwVJ9_qPizbtvt9OoPt39Gi4,3919
 validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py,sha256=aGsE1CfpfnNID631H15hnwAXtiDoy-HooIBMNuu8am4,4161
-validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py,sha256=YjDEVZNHuyQuJ2xC5e7fEkIlxa4Dj02z7SZpRr3yhes,5763
-validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py,sha256=5urEK4DAC8h8aKKDLrVF7xyF75dZxt0x3zYoejWdyZk,4651
-validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py,sha256=DM7HrUKhqVu0oXsrxz4KG1yM6u6b6wEdvMyPwotk0hM,5759
+validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py,sha256=pDN9aAjPNsHwo94biDtGBBJk7Daqv9UELIu8lCA_ZAs,5764
+validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py,sha256=SsFgUYPG8M8RbpcQKZzlx6aG4dG9yWK1zW3fgK52OUc,4652
+validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py,sha256=NDQeTt2Y3r241y3pUYhnkKtAhkOJZmE-8O6xxcW55dI,5760
 validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py,sha256=roCTA0QXU8GJoQIQ9X-FKeQ2LjBGwyH5IJTdZFnreRU,4645
-validmind/tests/model_validation/embeddings/utils.py,sha256=jVfPEwJBvtDMU5pC1zK7uEwQNNZd2hmMrZ3HP8jRaN8,1970
+validmind/tests/model_validation/embeddings/utils.py,sha256=Hr8jpVB0YfaOEYsO_tiwhU1UgXoJFHHlRqFcHDNXHoU,1896
 validmind/tests/model_validation/ragas/AnswerCorrectness.py,sha256=P1-sAWe4KGwgwJ0F44BY9Nd1wvCmaftG6tRZwsi9ero,5496
 validmind/tests/model_validation/ragas/AspectCritic.py,sha256=KsETxsVxSYAxAMbxyssL-c1AU0qUOFMz7I5D2pu-fY0,6936
 validmind/tests/model_validation/ragas/ContextEntityRecall.py,sha256=dQCBtiq4BF-C2covBaNxU0Ok_FYTDEN3M0U6ynwsAq0,5328
@@ -211,7 +206,7 @@ validmind/tests/model_validation/ragas/Faithfulness.py,sha256=SZCrgzfpcatfva9sYL
 validmind/tests/model_validation/ragas/NoiseSensitivity.py,sha256=oAcwDhXDEYLT0q3e8quHuYWdp7OJg9ycBvu05k6u6JI,6506
 validmind/tests/model_validation/ragas/ResponseRelevancy.py,sha256=vsfhXy-DBtIWEGsJZeR9xcE2V6-eLR2ui-qO1-96BLQ,5588
 validmind/tests/model_validation/ragas/SemanticSimilarity.py,sha256=wjEkCikFLweTdz3Tg5CzikWqusdSsWMwaD_4yntBCb4,4820
-validmind/tests/model_validation/ragas/utils.py,sha256=VCc3NcNLIwrYQ7RvuJ1ev4XhI86TKDVNzI8o12aHFHc,3363
+validmind/tests/model_validation/ragas/utils.py,sha256=EV3aHvWMOgVs6EIRS0nfOa7BRwpF-eSqlOQfsoZFTC8,3429
 validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py,sha256=5kk-etKeA7jfo6twQ4JVPEuNvWh0TBhhXUQL7SkrrWM,2858
 validmind/tests/model_validation/sklearn/AdjustedRandIndex.py,sha256=NCUM80CIFrV4Qm0P0wxMdf20y-BwLnPEJxOiPtv1eGk,2706
 validmind/tests/model_validation/sklearn/CalibrationCurve.py,sha256=-chYMKediopXV7YVNuaGhtb-RpIdgEsEpL1iy8aAgp4,4159
@@ -228,7 +223,7 @@ validmind/tests/model_validation/sklearn/HyperParametersTuning.py,sha256=Z7ibpCy
 validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py,sha256=wisUO4461arN6YPQ2lazbgnI2cOVjbSiDObt3T8IHYY,5279
 validmind/tests/model_validation/sklearn/MinimumAccuracy.py,sha256=2FVtoEMUJJYUxDW6WwC5agAojtt7FUnO7nwcVaqPKao,2773
 validmind/tests/model_validation/sklearn/MinimumF1Score.py,sha256=CBOGD_wCqcHgMbKfp5TGTc1H8mJoG3RwMRSOUFHVGDc,3069
-validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py,sha256=LyL2fde8pDcKozSUcDOd_IQF3AI8Z6jaq-wxL3guw2k,3624
+validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py,sha256=rNPD-qxdFKuUs8SWa0yIFIjjt8svPeXWNKWQrLy8uJc,3497
 validmind/tests/model_validation/sklearn/ModelParameters.py,sha256=CF3cZGJLxiABnf1CQ_u_iX_ylgvpElH3jF2DBXbXZJY,3060
 validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.py,sha256=wDxGUXgfzLA80wfjoRz7CzHO8NiQfuJyxIfuVFOuLYA,4658
 validmind/tests/model_validation/sklearn/OverfitDiagnosis.py,sha256=tpPf4lL_mfpMne6ei80QEYUZd1ual8w4SbXTKS_0nfY,9960
@@ -261,7 +256,7 @@ validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py,sh
 validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py,sha256=XpzPACbdKkjP5egxESDUYb7aCZ8_VmJpMHCy3joEHmA,3648
 validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py,sha256=4khg8CsgMWAQZVDOqky7_gcPqDS19d19n0FTTbh1cGM,3127
 validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py,sha256=UbtvxJho7yISQdsNcHOjUZwBASTY6TQmIpwLddYEsYo,4704
-validmind/tests/model_validation/statsmodels/RegressionModelSummary.py,sha256=ohX9OEVdofir7oK1rAKxbSImaxcsx_ZvjBk33piQAeA,2419
+validmind/tests/model_validation/statsmodels/RegressionModelSummary.py,sha256=3ljShlVkblJ2RNKSaTvsJ9QtdQGac-0ZcP1ne1bq94U,2354
 validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py,sha256=04vCwLMDUguS0z0Krbk9ubzrF5HVq0GGe9JqvKA7qQ4,3915
 validmind/tests/model_validation/statsmodels/ScorecardHistogram.py,sha256=0hnB6icasRKT_Cl0YxMEpIuaUKgi5scXHmV_nP9RmkI,4650
 validmind/tests/model_validation/statsmodels/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -290,8 +285,8 @@ validmind/tests/prompt_validation/NegativeInstruction.py,sha256=qGh_hI21zBfffSjD
 validmind/tests/prompt_validation/Robustness.py,sha256=4FShTFGzHcfg9ePfd_lbVt28MOmvlQlL7lF2VZALO2U,5695
 validmind/tests/prompt_validation/Specificity.py,sha256=nBKoBvE4vFisXNZX085h0rJN_m1cS9c05J4Ku53tuwQ,4790
 validmind/tests/prompt_validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-validmind/tests/prompt_validation/ai_powered_test.py,sha256=Lc8WU-rJ50e_NbLGV3YZ-W9t6Vj2T-o7hMxZbUrv3pw,2229
-validmind/tests/run.py,sha256=sO1YUFcM7Ng3yuVdC9cOR-7iOqmJF5EiE7gfuAateZg,13497
+validmind/tests/prompt_validation/ai_powered_test.py,sha256=pVmJPZVw1t0a3AMcGvBYEDR_sqI1Yu0AZjbshqBUVIg,2302
+validmind/tests/run.py,sha256=eIdR8If94ctaY25sWfygsLRi_0XoeJjqSZfdivocKU8,13703
 validmind/tests/test_providers.py,sha256=BceVuM_-bfQ4Zp-a5wwcP_wHeM6IOUpPIq1-MeT2-VY,6250
 validmind/tests/utils.py,sha256=mQuf1qgewPiE_pFN8iOoPSCGdyFqb4jbMFBVN3S3S2o,3526
 validmind/unit_metrics/__init__.py,sha256=lXeTJh8uq0TBRQHDBVhzKiHoV2eG9xOkHkI_pDXnkPU,952
@@ -311,7 +306,7 @@ validmind/unit_metrics/regression/MeanSquaredError.py,sha256=h-zgtlR3aigQwMGbi55
 validmind/unit_metrics/regression/QuantileLoss.py,sha256=rs0m9w4zIL6daQOHqYY-sEeQs6SDTpd0t3cN_KFZyqA,518
 validmind/unit_metrics/regression/RSquaredScore.py,sha256=z8-E-KSewvma9nu1OSUv97IfmFLpV5-rOq15jtlxklg,459
 validmind/unit_metrics/regression/RootMeanSquaredError.py,sha256=uIDsSpy75Z7W3zu4LditvW3mPJIkGxf-PdFQ7szWBZU,603
-validmind/utils.py,sha256=HXau6k_NkG4gYdtVQEn2h7P9mo0RzPpRWAKZUTOzXGk,18531
+validmind/utils.py,sha256=3T0uv_6cH0T_48utYvz9xWFPl-5oa0E74RxG5FsreJc,19470
 validmind/vm_models/__init__.py,sha256=lcqf9q2aRzrVrNN6R--81IkrnSa6BXPbhJ8SnkT_hcI,702
 validmind/vm_models/dataset/__init__.py,sha256=U4CxZjdoc0dd9u2AqBl5PJh1UVbzXWNrmundmjLF-qE,346
 validmind/vm_models/dataset/dataset.py,sha256=F6_rc5pjccRLnB7UcIMiGMbD-qMVUW5v4TnZTNSXTbo,26370
@@ -321,14 +316,14 @@ validmind/vm_models/input.py,sha256=qLdqz_bktr4v0YcPha2vFdDvmkC-btT1pH9zBIkt1OY,
 validmind/vm_models/model.py,sha256=PRNyrnKihIRtbYt4idLPHf8OCij71Vgc5Xug_oVZfBg,6486
 validmind/vm_models/result/__init__.py,sha256=Bs5GMGDxiTsxlwCdqxz5LmGkY0_fM6-_0-3tWSRoqps,341
 validmind/vm_models/result/result.jinja,sha256=Yvovwm5gInCBukFRlvJXNlDIUpl2eFz4dz1lS3Sn_Gc,311
-validmind/vm_models/result/result.py,sha256=kMyr_7qOgo30ZW87MomIm-ck5d8Ph2Kx4cUE2hh-EEM,13784
+validmind/vm_models/result/result.py,sha256=McmQnFiSUL4zXxB1u2-saoCkTxPMUTQZEtk3ZpmpyEg,15266
 validmind/vm_models/result/utils.py,sha256=t6g-g1fJ3SU9lHqC1kMeozMkrUnfOMwYAep3Z5XFXNo,5122
 validmind/vm_models/test_suite/runner.py,sha256=Cpl9WKwHzJD5Zvrh71FzbEhGZkHM0x0MSd4PIwdOLDQ,5427
 validmind/vm_models/test_suite/summary.py,sha256=Ug3nMvpPL2DSTDujWagWMCrFiW9oDy0AqJL_zXN8pH0,4642
 validmind/vm_models/test_suite/test.py,sha256=uImjmPlBlLrlVPavsUzbaDK55bvpOn3PuFyWeyYyTac,3908
 validmind/vm_models/test_suite/test_suite.py,sha256=5Jppt2UXSMgvJ6FO5LIAKA4oN_-hh9SMr8APAFJzk9g,5080
-validmind-2.7.9.dist-info/LICENSE,sha256=XonPUfwjvrC5Ombl3y-ko0Wubb1xdG_7nzvIbkZRKHw,35772
-validmind-2.7.9.dist-info/METADATA,sha256=jzQS-5FdwOi9PjQmKrHKhfh9xmdri5w5DlFK6Yeex9c,6084
-validmind-2.7.9.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-validmind-2.7.9.dist-info/entry_points.txt,sha256=HuW7YyOv9u_OEWpViQXtv0nfoI67uieJHawKWA4Hv9A,76
-validmind-2.7.9.dist-info/RECORD,,
+validmind-2.8.10.dist-info/LICENSE,sha256=XonPUfwjvrC5Ombl3y-ko0Wubb1xdG_7nzvIbkZRKHw,35772
+validmind-2.8.10.dist-info/METADATA,sha256=MJDPX24c6xbCA6iNdyA2096lGY2rm6ip1KfBkWLd12A,6058
+validmind-2.8.10.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+validmind-2.8.10.dist-info/entry_points.txt,sha256=HuW7YyOv9u_OEWpViQXtv0nfoI67uieJHawKWA4Hv9A,76
+validmind-2.8.10.dist-info/RECORD,,

{validmind-2.7.9.dist-info → validmind-2.8.10.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.0.1
+Generator: poetry-core 2.1.1
 Root-Is-Purelib: true
 Tag: py3-none-any

validmind/ai/test_result_description/config.yaml

@@ -1,29 +0,0 @@
-id: test_result_description
-name: Test Result Description
-description: Generate a description for a test result
-version: 0.1.0
-model: gpt-4o
-temperature: 0.0
-output_type: markdown
-prompts:
-  system:
-    role: system
-    path: system.jinja
-  user:
-    role: user
-    path: user.jinja
-inputs:
-  test_name:
-    description: The name of the test that produced the result (usually the last part of the test ID)
-    type: string
-  test_description:
-    description: The description (docstring) of the test that was run
-    type: string
-  summary:
-    description: The json result summary (i.e. the table(s) returned by the test)
-    type: list
-    optional: true
-  figures:
-    description: A list of base64 encoded images of the figures returned by the test
-    type: list
-    optional: true

validmind/ai/test_result_description/context.py

@@ -1,73 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-import multiprocessing
-
-MIN_IMAGES_FOR_PARALLEL = 4
-MAX_WORKERS = multiprocessing.cpu_count()
-
-
-def parallel_downsample_images(base64_strings):
-    import os
-    import sys
-
-    sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
-    from test_result_description.image_processing import (
-        downsample_image,  # type: ignore
-    )
-
-    num_images = len(base64_strings)
-
-    if num_images < MIN_IMAGES_FOR_PARALLEL:
-        return [downsample_image(img) for img in base64_strings]
-
-    num_workers = min(num_images, MAX_WORKERS)
-
-    with multiprocessing.Pool(processes=num_workers) as pool:
-        results = pool.map(downsample_image, base64_strings)
-
-    sys.path.pop(0)
-
-    return results
-
-
-class Context:
-    def __init__(self, mode="local"):
-        pass
-
-    def load(self, input_data):
-        # this task can accept a dict or a test result object from the ValidMind Library
-        if isinstance(input_data, dict):
-            return input_data
-
-        # we are likely running outside of the ValidMind Library and need to convert
-        # the test result object to a dictionary
-        test_result = input_data
-
-        try:
-            from markdownify import markdownify as md
-        except ImportError as e:
-            raise ImportError(
-                "Failed to import markdownify. Please install the package to use this task."
-            ) from e
-
-        input_data = {
-            "test_name": test_result.result_id.split(".")[-1],
-            "test_description": md(test_result.result_metadata[0]["text"]),
-        }
-
-        if hasattr(test_result, "metric") and test_result.metric.summary is not None:
-            input_data["summary"] = test_result.metric.summary.serialize()
-        elif (
-            hasattr(test_result, "test_results")
-            and test_result.test_results.summary is not None
-        ):
-            input_data["summary"] = test_result.test_results.summary.serialize()
-
-        if test_result.figures:
-            input_data["figures"] = parallel_downsample_images(
-                [figure._get_b64_url() for figure in test_result.figures]
-            )
-
-        return input_data

validmind/ai/test_result_description/image_processing.py

@@ -1,124 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-import base64
-import io
-
-import numpy as np
-from PIL import Image, ImageEnhance, ImageFilter
-
-DOWNSAMPLE_PERCENTAGE = 50
-
-
-def open_base64_image(base64_string):
-    if base64_string.startswith("data:image/png;base64,"):
-        base64_string = base64_string.split(",")[1]
-
-    image_data = base64.b64decode(base64_string)
-    image_buffer = io.BytesIO(image_data)
-    image = Image.open(image_buffer)
-
-    return image
-
-
-def downsample_image(base64_string):
-    image = open_base64_image(base64_string)
-
-    # Calculate the target dimensions based on the reduction percentage
-    target_width = int(image.width * (1 - DOWNSAMPLE_PERCENTAGE / 100))
-    target_height = int(image.height * (1 - DOWNSAMPLE_PERCENTAGE / 100))
-
-    # If the image is already smaller than the target size, return the original
-    if image.width <= target_width and image.height <= target_height:
-        return base64_string
-
-    # remove any margins from the image
-    # Find the bounding box of non-uniform pixels (margin detection)
-    width, height = image.size
-    background = image.getpixel((0, 0))  # Assume top-left pixel is background color
-
-    def is_different(pixel):
-        return pixel != background
-
-    left = next(
-        x
-        for x in range(width)
-        if any(is_different(image.getpixel((x, y))) for y in range(height))
-    )
-    right = next(
-        x
-        for x in range(width - 1, -1, -1)
-        if any(is_different(image.getpixel((x, y))) for y in range(height))
-    )
-    top = next(
-        y
-        for y in range(height)
-        if any(is_different(image.getpixel((x, y))) for x in range(width))
-    )
-    bottom = next(
-        y
-        for y in range(height - 1, -1, -1)
-        if any(is_different(image.getpixel((x, y))) for x in range(width))
-    )
-
-    # Crop the image to remove the uniform margin (with some padding)
-    bbox = (left - 5, top - 5, right + 6, bottom + 6)
-    image = image.crop(bbox)
-
-    # If the image has an alpha channel, remove any transparent margins
-    if image.mode in ("RGBA", "LA"):
-        alpha = image.getchannel("A")
-        bbox = alpha.getbbox()
-        if bbox:
-            image = image.crop(bbox)
-
-    # Apply unsharp mask to enhance edges
-    image = image.filter(ImageFilter.UnsharpMask(radius=2, percent=150, threshold=3))
-
-    # Calculate new dimensions
-    aspect_ratio = image.width / image.height
-    new_height = target_height
-    new_width = int(new_height * aspect_ratio)
-
-    # print(f"downsampling from {width}x{height} to {new_width}x{new_height}")
-
-    # Ensure we don't exceed the target width
-    if new_width > target_width:
-        new_width = target_width
-        new_height = int(new_width / aspect_ratio)
-
-    # print(f"downsampling from {image.width}x{image.height} to {new_width}x{new_height}")
-
-    # Convert to numpy array for custom downsampling
-    img_array = np.array(image)
-
-    # Optimized area interpolation
-    h_factor = img_array.shape[0] / new_height
-    w_factor = img_array.shape[1] / new_width
-
-    h_indices = (np.arange(new_height).reshape(-1, 1) * h_factor).astype(int)
-    w_indices = (np.arange(new_width).reshape(1, -1) * w_factor).astype(int)
-
-    h_indices = np.minimum(h_indices, img_array.shape[0] - 1)
-    w_indices = np.minimum(w_indices, img_array.shape[1] - 1)
-
-    # Convert back to PIL Image
-    image = Image.fromarray(img_array[h_indices, w_indices].astype(np.uint8))
-
-    # Enhance contrast slightly
-    enhancer = ImageEnhance.Contrast(image)
-    image = enhancer.enhance(1.2)
-
-    # Sharpen the image
-    image = image.filter(ImageFilter.SHARPEN)
-
-    # Convert the image to bytes in PNG format
-    buffered = io.BytesIO()
-    image.save(buffered, format="PNG")
-    img_bytes = buffered.getvalue()
-
-    # Encode the bytes to base64
-    b64_encoded = base64.b64encode(img_bytes).decode("utf-8")
-
-    return f"data:image/png;base64,{b64_encoded}"

validmind/ai/test_result_description/system.jinja

@@ -1,39 +0,0 @@
-You are an expert data scientist and MRM specialist.
-You are tasked with analyzing the results of a quantitative test run on some model or dataset.
-Your goal is to create a test description that will act as part of the model documentation.
-You will provide both the developer and other consumers of the documentation with a clear and concise "interpretation" of the results they will see.
-The overarching theme to maintain is MRM documentation.
-
-Examine the provided statistical test results and compose a description of the results.
-The results are either in the form of serialized tables or images of plots.
-Compose a description and interpretation of the result to accompany it in MRM documentation.
-It will be read by other data scientists and developers and by validators and stakeholders.
-
-Use valid Markdown syntax to format the response.
-Avoid long sentences and complex vocabulary.
-Avoid overly verbose explanations - the goal is to explain to a user what they are seeing in the results.
-Structure the response clearly and logically.
-Respond only with your analysis and insights, not the verbatim test results.
-Respond only with the markdown content, no explanation or context for your response is necessary.
-Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
-
-Explain the test, its purpose, its mechanism/formula etc and why it is useful.
-If relevant, provide a very brief description of the way this test is used in model/dataset evaluation and how it is interpreted.
-Highlight the key insights from the test results. The key insights should be concise and easily understood.
-An insight should only be included if it is something not entirely obvious from the test results.
-End the response with any closing remarks, summary or additional useful information.
-
-Use the following format for the response (feel free to stray from it if necessary - this is a suggested starting point):
-
-<ResponseFormat>
-**<Test Name>** calculates the xyz <continue to explain what it does in detail>...
-
-This test is useful for <explain why and for what this test is useful>...
-
-**Key Insights:**
-
-The following key insights can be identified in the test results:
-
-- **<key insight 1 - title>**: <concise explanation of key insight 1>
-- ...<continue with any other key insights using the same format>
-</ResponseFormat>

validmind/ai/test_result_description/user.jinja

@@ -1,30 +0,0 @@
-**Test ID**: `{{ test_name }}`
-
-**Test Description**:
-
-{{ test_description }}
-
----
-
-Generate a description of the following result of the test using the instructions given in your system prompt.
-
-{%- if context %}
-**Context**:
-{{ context }}
-{%- endif %}
-
-{%- if summary %}
-**Test Result Tables** *(Raw Data)*:
-{{ summary }}
-{%- endif %}
-
-{%- if figures %}
-The following images make up the results of the test.
-{%- for b64_image_url in figures %}
-[[IMAGE:{{ b64_image_url }}]]
-{%- endfor %}
-{%- endif %}
-
-Keep your response concise and to the point!
-Only include content in your response if its something truly insightful or interesting!
-DO NOT VERBOSELY EXPLAIN THE TEST OR THE RESULTS!!!