validmind 2.7.9__tar.gz → 2.8.10__tar.gz

{validmind-2.7.9 → validmind-2.8.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: validmind
-Version: 2.7.9
+Version: 2.8.10
 Summary: ValidMind Library
 License: Commercial License
 Author: Andres Rodriguez
@@ -15,8 +15,8 @@ Provides-Extra: all
 Provides-Extra: huggingface
 Provides-Extra: llm
 Provides-Extra: pytorch
-Provides-Extra: r-support
 Requires-Dist: aiohttp[speedups]
+Requires-Dist: anywidget (>=0.9.13,<0.10.0)
 Requires-Dist: arch
 Requires-Dist: bert-score (>=0.3.13)
 Requires-Dist: catboost
@@ -42,7 +42,6 @@ Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"
 Requires-Dist: python-dotenv
 Requires-Dist: ragas (>=0.2.3) ; extra == "all" or extra == "llm"
 Requires-Dist: rouge (>=1)
-Requires-Dist: rpy2 (>=3.5.10,<4.0.0) ; extra == "all" or extra == "r-support"
 Requires-Dist: scikit-learn (<1.6.0)
 Requires-Dist: scipy
 Requires-Dist: scorecardpy (>=0.1.9.6,<0.2.0.0)
@@ -53,6 +52,7 @@ Requires-Dist: shap (==0.44.1)
 Requires-Dist: statsmodels
 Requires-Dist: tabulate (>=0.8.9,<0.9.0)
 Requires-Dist: textblob (>=0.18.0.post0,<0.19.0)
+Requires-Dist: tiktoken
 Requires-Dist: torch (>=1.10.0) ; extra == "all" or extra == "llm" or extra == "pytorch"
 Requires-Dist: tqdm
 Requires-Dist: transformers (>=4.32.0,<5.0.0) ; extra == "all" or extra == "huggingface" or extra == "llm"
@@ -131,6 +131,7 @@ The ValidMind Library has optional dependencies that can be installed separately
 - **R Models**: To use R models with the ValidMind Library, install the `r` extra:
 
     ```bash
-    pip install validmind[r-support]
+    pip install validmind
+    pip install rpy2
     ```
 

{validmind-2.7.9 → validmind-2.8.10}/README.pypi.md

@@ -68,5 +68,6 @@ The ValidMind Library has optional dependencies that can be installed separately
 - **R Models**: To use R models with the ValidMind Library, install the `r` extra:
 
     ```bash
-    pip install validmind[r-support]
+    pip install validmind
+    pip install rpy2
     ```

{validmind-2.7.9 → validmind-2.8.10}/pyproject.toml

@@ -10,10 +10,11 @@ description = "ValidMind Library"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.7.9"
+version = "2.8.10"
 
 [tool.poetry.dependencies]
 aiohttp = {extras = ["speedups"], version = "*"}
+anywidget = "^0.9.13"
 arch = "*"
 bert-score = ">=0.3.13"
 catboost = "*"
@@ -40,7 +41,6 @@ python = ">=3.8.1,<3.12"
 python-dotenv = "*"
 ragas = {version = ">=0.2.3", optional = true}
 rouge = ">=1"
-rpy2 = {version = "^3.5.10", optional = true}
 scikit-learn = "*,<1.6.0"
 scipy = "*"
 scorecardpy = "^0.1.9.6"
@@ -51,6 +51,7 @@ shap = "0.44.1"
 statsmodels = "*"
 tabulate = "^0.8.9"
 textblob = "^0.18.0.post0"
+tiktoken = "*"
 torch = {version = ">=1.10.0", optional = true}
 tqdm = "*"
 transformers = {version = "^4.32.0", optional = true}
@@ -76,7 +77,6 @@ twine = "^4.0.2"
 
 [tool.poetry.extras]
 all = [
-  "rpy2",
   "torch",
   "transformers",
   "pycocoevalcap",
@@ -94,7 +94,6 @@ llm = [
   "langchain-openai",
 ]
 pytorch = ["torch"]
-r-support = ["rpy2"]
 
 [build-system]
 build-backend = "poetry.core.masonry.api"

{validmind-2.7.9 → validmind-2.8.10}/validmind/__init__.py

@@ -30,8 +30,12 @@ vm.init(
 
 After you have pasted the code snippet into your development source code and executed the code, the Python Library API will register with ValidMind. You can now use the ValidMind Library to document and test your models, and to upload to the ValidMind Platform.
 """
+import threading
 import warnings
 
+import pkg_resources
+from IPython.display import HTML, display
+
 # Ignore Numba warnings. We are not requiring this package directly
 from numba.core.errors import NumbaDeprecationWarning, NumbaPendingDeprecationWarning
 
@@ -51,30 +55,74 @@ from .client import (  # noqa: E402
 )
 from .tests.decorator import tags, tasks, test
 from .tests.run import print_env
+from .utils import is_notebook, parse_version
 from .vm_models.result import RawData
 
+__shown = False
+
+
+def show_warning(installed, running):
+    global __shown
+
+    if __shown:
+        return
+    __shown = True
+
+    message = (
+        f"⚠️ This kernel is running an older version of validmind ({running}) "
+        f"than the latest version installed on your system ({installed}).\n\n"
+        "You may need to restart the kernel if you are experiencing issues."
+    )
+    display(HTML(f"<div style='color: red;'>{message}</div>"))
+
+
+def check_version():
+    # get the installed vs running version of validmind
+    # to make sure we are using the latest installed version
+    # in case user has updated the package but forgot to restart the kernel
+    installed = pkg_resources.get_distribution("validmind").version
+    running = __version__
+
+    if parse_version(installed) > parse_version(running):
+        show_warning(installed, running)
+
+    # Schedule the next check for 5 minutes from now
+    timer = threading.Timer(300, check_version)
+    timer.daemon = True
+    timer.start()
+
+
+if is_notebook():
+    check_version()
+
 __all__ = [  # noqa
     "__version__",
-    # Python Library API
-    "datasets",
-    "errors",
-    "get_test_suite",
+    # main library API
     "init",
+    "reload",
     "init_dataset",
     "init_model",
     "init_r_model",
     "preview_template",
-    "print_env",
-    "RawData",
-    "reload",
     "run_documentation_tests",
+    # log metric function (for direct/bulk/retroactive logging of metrics)
+    "log_metric",
+    # test suite functions (less common)
+    "get_test_suite",
     "run_test_suite",
+    # helper functions (for troubleshooting)
+    "print_env",
+    # decorators (for building tests
     "tags",
     "tasks",
     "test",
-    "tests",
-    "test_suites",
+    # raw data (for post-processing test results and building tests)
+    "RawData",
+    # submodules
+    "datasets",
+    "errors",
     "vm_models",
+    "tests",
     "unit_metrics",
-    "log_metric",
+    "test_suites",
 ]

validmind-2.8.10/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.8.10"

{validmind-2.7.9 → validmind-2.8.10}/validmind/ai/test_descriptions.py

@@ -4,70 +4,24 @@
 
 import json
 import os
-import re
 from concurrent.futures import ThreadPoolExecutor
 from typing import List, Optional, Union
 
 import tiktoken
-from jinja2 import Template
 
 from ..client_config import client_config
 from ..logging import get_logger
 from ..utils import NumpyEncoder, md_to_html, test_id_to_name
 from ..vm_models.figure import Figure
 from ..vm_models.result import ResultTable
-from .utils import DescriptionFuture, get_client_and_model
+from .utils import DescriptionFuture
 
 __executor = ThreadPoolExecutor()
-__prompt = None
 
 logger = get_logger(__name__)
 
 
-def _load_prompt():
-    global __prompt
-
-    if not __prompt:
-        folder_path = os.path.join(os.path.dirname(__file__), "test_result_description")
-        with open(os.path.join(folder_path, "system.jinja"), "r") as f:
-            system_prompt = f.read()
-        with open(os.path.join(folder_path, "user.jinja"), "r") as f:
-            user_prompt = f.read()
-
-        __prompt = (Template(system_prompt), Template(user_prompt))
-
-    return __prompt
-
-
-def prompt_to_message(role, prompt):
-    if "[[IMAGE:" not in prompt:
-        return {"role": role, "content": prompt}
-
-    content = []
-
-    # Regex pattern to find [[IMAGE:<b64-data>]] markers
-    pattern = re.compile(r"\[\[IMAGE:(.*?)\]\]", re.DOTALL)
-
-    last_index = 0
-    for match in pattern.finditer(prompt):
-        # Text before the image marker
-        start, end = match.span()
-        if start > last_index:
-            content.append({"type": "text", "text": prompt[last_index:start]})
-
-        content.append({"type": "image_url", "image_url": {"url": match.group(1)}})
-
-        last_index = end
-
-    # Text after the last image
-    if last_index < len(prompt):
-        content.append({"type": "text", "text": prompt[last_index:]})
-
-    return {"role": role, "content": content}
-
-
 def _get_llm_global_context():
-
     # Get the context from the environment variable
     context = os.getenv("VALIDMIND_LLM_DESCRIPTIONS_CONTEXT", "")
 
@@ -115,13 +69,13 @@ def generate_description(
     title: Optional[str] = None,
 ):
     """Generate the description for the test results"""
+    from validmind.api_client import generate_test_result_description
+
     if not tables and not figures and not metric:
         raise ValueError(
             "No tables, unit metric or figures provided - cannot generate description"
         )
 
-    client, model = get_client_and_model()
-
     # get last part of test id
     test_name = title or test_id.split(".")[-1]
 
@@ -145,29 +99,18 @@ def generate_description(
     else:
         summary = None
 
-    context = _get_llm_global_context()
-
-    input_data = {
-        "test_name": test_name,
-        "test_description": test_description,
-        "title": title,
-        "summary": _truncate_summary(summary, test_id),
-        "figures": [figure._get_b64_url() for figure in ([] if tables else figures)],
-        "context": context,
-    }
-    system, user = _load_prompt()
-
-    messages = [
-        prompt_to_message("system", system.render(input_data)),
-        prompt_to_message("user", user.render(input_data)),
-    ]
-    response = client.chat.completions.create(
-        model=model,
-        temperature=0.0,
-        messages=messages,
-    )
-
-    return response.choices[0].message.content
+    return generate_test_result_description(
+        {
+            "test_name": test_name,
+            "test_description": test_description,
+            "title": title,
+            "summary": _truncate_summary(summary, test_id),
+            "figures": [
+                figure._get_b64_url() for figure in ([] if tables else figures)
+            ],
+            "context": _get_llm_global_context(),
+        }
+    )["content"]
 
 
 def background_generate_description(
@@ -238,7 +181,8 @@ def get_result_description(
     # Check the feature flag first, then the environment variable
     llm_descriptions_enabled = (
         client_config.can_generate_llm_test_descriptions()
-        and os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1") not in ["0", "false"]
+        and os.getenv("VALIDMIND_LLM_DESCRIPTIONS_ENABLED", "1").lower()
+        not in ["0", "false"]
     )
 
     # TODO: fix circular import

{validmind-2.7.9 → validmind-2.8.10}/validmind/api_client.py

@@ -194,6 +194,7 @@ def init(
     api_host: Optional[str] = None,
     model: Optional[str] = None,
     monitoring: bool = False,
+    generate_descriptions: Optional[bool] = None,
 ):
     """
     Initializes the API client instances and calls the /ping endpoint to ensure
@@ -209,7 +210,7 @@ def init(
         api_secret (str, optional): The API secret. Defaults to None.
         api_host (str, optional): The API host. Defaults to None.
         monitoring (bool): The ongoing monitoring flag. Defaults to False.
-
+        generate_descriptions (bool): Whether to use GenAI to generate test result descriptions. Defaults to True.
     Raises:
         ValueError: If the API key and secret are not provided
     """
@@ -235,6 +236,9 @@ def init(
 
     _monitoring = monitoring
 
+    if generate_descriptions is not None:
+        os.environ["VALIDMIND_LLM_DESCRIPTIONS_ENABLED"] = str(generate_descriptions)
+
     reload()
 
 
@@ -487,3 +491,16 @@ def get_ai_key() -> Dict[str, Any]:
         raise_api_error(r.text)
 
     return r.json()
+
+
+def generate_test_result_description(test_result_data: Dict[str, Any]) -> str:
+    r = requests.post(
+        url=_get_url("ai/generate/test_result_description"),
+        headers=_get_api_headers(),
+        json=test_result_data,
+    )
+
+    if r.status_code != 200:
+        raise_api_error(r.text)
+
+    return r.json()

{validmind-2.7.9 → validmind-2.8.10}/validmind/errors.py

@@ -228,7 +228,7 @@ class MissingRExtrasError(BaseError):
     def description(self, *args, **kwargs):
         return (
             self.message
-            or "ValidMind r-support needs to be installed: `pip install validmind[r-support]`"
+            or "`rpy2` is required to use R models. Please install it with `pip install rpy2`"
         )
 
 

{validmind-2.7.9 → validmind-2.8.10}/validmind/models/r_model.py

@@ -5,6 +5,7 @@
 import numpy as np
 import pandas as pd
 
+from validmind.errors import MissingRExtrasError
 from validmind.logging import get_logger
 from validmind.vm_models.model import VMModel
 
@@ -125,7 +126,10 @@ class RModel(VMModel):
         """
         Converts the predicted probabilities to classes
         """
-        from rpy2.robjects import pandas2ri
+        try:
+            from rpy2.robjects import pandas2ri
+        except ImportError:
+            raise MissingRExtrasError()
 
         # Activate the pandas conversion for rpy2
         pandas2ri.activate()

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/comparison.py

@@ -15,7 +15,7 @@ from validmind.vm_models.figure import (
     is_png_image,
 )
 from validmind.vm_models.input import VMInput
-from validmind.vm_models.result import ResultTable, TestResult
+from validmind.vm_models.result import RawData, ResultTable, TestResult
 
 logger = get_logger(__name__)
 
@@ -312,6 +312,25 @@ def get_comparison_test_configs(
     return test_configs
 
 
+def _combine_raw_data(results: List[TestResult]) -> RawData:
+    """Combine RawData objects"""
+    attribute_names = results[0].raw_data.__dict__.keys()
+
+    # check that all the raw data objects have the same attributes
+    for result in results:
+        if not isinstance(result.raw_data, RawData):
+            raise ValueError("All raw data objects must be of type RawData")
+        if result.raw_data.__dict__.keys() != attribute_names:
+            raise ValueError("RawData objects must have the same attributes")
+
+    return RawData(
+        **{
+            key: [getattr(result.raw_data, key) for result in results]
+            for key in attribute_names
+        }
+    )
+
+
 def combine_results(
     results: List[TestResult],
 ) -> Tuple[List[Any], Dict[str, List[Any]], Dict[str, List[Any]]]:
@@ -338,6 +357,9 @@ def combine_results(
     # handle threshold tests (i.e. tests that have pass/fail bool status)
     if results[0].passed is not None:
         combined_outputs.append(all(result.passed for result in results))
+    # handle raw data (if any)
+    if results[0].raw_data:
+        combined_outputs.append(_combine_raw_data(results))
 
     # combine inputs and params
     combined_inputs = {}
@@ -359,4 +381,8 @@ def combine_results(
     combined_inputs = _combine_dict_values(combined_inputs)
     combined_params = _combine_dict_values(combined_params)
 
-    return combined_outputs, combined_inputs, combined_params
+    return (
+        tuple(combined_outputs),
+        combined_inputs,
+        combined_params,
+    )

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/data_validation/FeatureTargetCorrelationPlot.py

@@ -52,16 +52,10 @@ def FeatureTargetCorrelationPlot(dataset, fig_height=600):
     - Not apt for models that employ complex feature interactions, like Decision Trees or Neural Networks, as the test
     may not accurately reflect their importance.
     """
-
-    # Filter DataFrame based on features and target_column
     df = dataset.df[dataset.feature_columns + [dataset.target_column]]
 
-    fig = _visualize_feature_target_correlation(df, dataset.target_column, fig_height)
-
-    correlations = (
-        df.corr(numeric_only=True)[dataset.target_column]
-        .drop(dataset.target_column)
-        .to_frame()
+    fig, correlations = _visualize_feature_target_correlation(
+        df, dataset.target_column, fig_height
     )
 
     return fig, RawData(correlation_data=correlations)
@@ -100,4 +94,5 @@ def _visualize_feature_target_correlation(df, target_column, fig_height):
         yaxis_title="",
         height=fig_height,  # Adjust the height value as needed
     )
-    return fig
+
+    return fig, correlations

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/model_validation/ContextualRecall.py

@@ -118,4 +118,4 @@ def ContextualRecall(dataset, model):
     # Create a DataFrame from all collected statistics
     result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
 
-    return (result_df, *tuple(figures), RawData(contextual_recall_scores=metrics_df))
+    return (result_df, *figures, RawData(contextual_recall_scores=metrics_df))

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/model_validation/MeteorScore.py

@@ -117,4 +117,4 @@ def MeteorScore(dataset, model):
     # Create a DataFrame from all collected statistics
     result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
 
-    return (result_df, *tuple(figures), RawData(meteor_scores=metrics_df))
+    return (result_df, *figures, RawData(meteor_scores=metrics_df))

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/model_validation/ToxicityScore.py

@@ -141,7 +141,7 @@ def ToxicityScore(dataset, model):
 
     return (
         result_df,
-        *tuple(figures),
+        *figures,
         RawData(
             input_toxicity_df=input_df,
             true_toxicity_df=true_df,

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py

@@ -151,4 +151,4 @@ def StabilityAnalysisRandomNoise(
         mean_similarity_threshold,
     )
 
-    return result, RawData(original_perturbed_similarity=raw_data)
+    return *result, RawData(original_perturbed_similarity=raw_data)

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py

@@ -107,4 +107,4 @@ def StabilityAnalysisSynonyms(
         mean_similarity_threshold,
     )
 
-    return result, RawData(original_perturbed_similarity=raw_data)
+    return *result, RawData(original_perturbed_similarity=raw_data)

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py

@@ -134,4 +134,4 @@ def StabilityAnalysisTranslation(
         mean_similarity_threshold,
     )
 
-    return result, RawData(original_perturbed_similarity=raw_data)
+    return *result, RawData(original_perturbed_similarity=raw_data)

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/model_validation/embeddings/utils.py

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import numpy as np
-import pandas as pd
 import plotly.express as px
 from sklearn.metrics.pairwise import cosine_similarity
 
@@ -18,14 +17,12 @@ def create_stability_analysis_result(
         original_embeddings, perturbed_embeddings
     ).diagonal()
 
-    # create a raw dataframe of the original, perturbed and similarity
-    raw_data = pd.DataFrame(
-        {
-            "original": original_embeddings,
-            "perturbed": perturbed_embeddings,
-            "similarity": similarities,
-        }
-    )
+    # Store raw data in a dictionary
+    raw_data = {
+        "original_embeddings": original_embeddings,
+        "perturbed_embeddings": perturbed_embeddings,
+        "similarities": similarities,
+    }
 
     mean = np.mean(similarities)
     passed = mean > mean_similarity_threshold

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/model_validation/ragas/utils.py

@@ -4,24 +4,25 @@
 
 import os
 
-from validmind.ai.utils import get_client_and_model
-from validmind.client_config import client_config
+from validmind.ai.utils import get_client_and_model, is_configured
 
 EMBEDDINGS_MODEL = "text-embedding-3-small"
 
 
 def get_ragas_config():
-    if not client_config.can_generate_llm_test_descriptions():
-        raise ValueError(
-            "LLM based descriptions are not enabled in the current configuration."
-        )
-
     # import here since its an optional dependency
     try:
         from langchain_openai import ChatOpenAI, OpenAIEmbeddings
     except ImportError:
         raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
 
+    if not is_configured():
+        raise ValueError(
+            "LLM is not configured. Please set an `OPENAI_API_KEY` environment variable "
+            "or ensure that you are connected to the ValidMind API and ValidMind AI is "
+            "enabled for your account."
+        )
+
     client, model = get_client_and_model()
     os.environ["OPENAI_API_BASE"] = str(client.base_url)
 

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py

@@ -62,18 +62,14 @@ def MinimumROCAUCScore(dataset: VMDataset, model: VMModel, min_threshold: float
         lb = LabelBinarizer()
         lb.fit(y_true)
 
-        y_true_binarized = lb.transform(y_true)
-        y_score_binarized = lb.transform(dataset.y_pred(model))
-
         roc_auc = roc_auc_score(
-            y_true=y_true_binarized,
-            y_score=y_score_binarized,
+            y_true=lb.transform(y_true),
+            y_score=lb.transform(dataset.y_pred(model)),
             average="macro",
         )
 
     else:
-        y_score_prob = dataset.y_prob(model)
-        roc_auc = roc_auc_score(y_true=y_true, y_score=y_score_prob)
+        roc_auc = roc_auc_score(y_true=y_true, y_score=dataset.y_prob(model))
 
     return [
         {

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/model_validation/statsmodels/RegressionModelSummary.py

@@ -45,17 +45,17 @@ def RegressionModelSummary(dataset: VMDataset, model: VMModel):
     - A high R-Squared or Adjusted R-Squared may not necessarily indicate a good model, especially in cases of
     overfitting.
     """
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
     return [
         {
             "Independent Variables": dataset.feature_columns,
-            "R-Squared": r2_score(dataset.y, dataset.y_pred(model)),
+            "R-Squared": r2_score(y_true, y_pred),
             "Adjusted R-Squared": adj_r2_score(
-                dataset.y,
-                dataset.y_pred(model),
-                len(dataset.y),
-                len(dataset.feature_columns),
+                y_true, y_pred, len(y_true), len(dataset.feature_columns)
             ),
-            "MSE": mean_squared_error(dataset.y, dataset.y_pred(model), squared=True),
-            "RMSE": mean_squared_error(dataset.y, dataset.y_pred(model), squared=False),
+            "MSE": mean_squared_error(y_true, y_pred, squared=True),
+            "RMSE": mean_squared_error(y_true, y_pred, squared=False),
         }
     ]

{validmind-2.7.9 → validmind-2.8.10}/validmind/tests/prompt_validation/ai_powered_test.py

@@ -4,8 +4,7 @@
 
 import re
 
-from validmind.ai.utils import get_client_and_model
-from validmind.client_config import client_config
+from validmind.ai.utils import get_client_and_model, is_configured
 
 missing_prompt_message = """
 Cannot run prompt validation tests on a model with no prompt.
@@ -25,9 +24,11 @@ def call_model(
     system_prompt: str, user_prompt: str, temperature: float = 0.0, seed: int = 42
 ):
     """Call LLM with the given prompts and return the response"""
-    if not client_config.can_generate_llm_test_descriptions():
+    if not is_configured():
         raise ValueError(
-            "LLM based descriptions are not enabled for your organization."
+            "LLM is not configured. Please set an `OPENAI_API_KEY` environment variable "
+            "or ensure that you are connected to the ValidMind API and ValidMind AI is "
+            "enabled for your account."
         )
 
     client, model = get_client_and_model()