validmind 2.8.29__py3-none-any.whl → 2.10.0rc1__py3-none-any.whl

validmind/__init__.py

@@ -32,15 +32,21 @@ After you have pasted the code snippet into your development source code and exe
 """
 import threading
 import warnings
+from importlib import metadata
 
-import pkg_resources
 from IPython.display import HTML, display
 
 # Ignore Numba warnings. We are not requiring this package directly
-from numba.core.errors import NumbaDeprecationWarning, NumbaPendingDeprecationWarning
+try:
+    from numba.core.errors import (
+        NumbaDeprecationWarning,
+        NumbaPendingDeprecationWarning,
+    )
 
-warnings.simplefilter("ignore", category=NumbaDeprecationWarning)
-warnings.simplefilter("ignore", category=NumbaPendingDeprecationWarning)
+    warnings.simplefilter("ignore", category=NumbaDeprecationWarning)
+    warnings.simplefilter("ignore", category=NumbaPendingDeprecationWarning)
+except ImportError:
+    ...
 
 from .__version__ import __version__  # noqa: E402
 from .api_client import init, log_metric, log_text, reload
@@ -81,7 +87,12 @@ def check_version():
     # get the installed vs running version of validmind
     # to make sure we are using the latest installed version
     # in case user has updated the package but forgot to restart the kernel
-    installed = pkg_resources.get_distribution("validmind").version
+    try:
+        installed = metadata.version("validmind")
+    except metadata.PackageNotFoundError:
+        # Package metadata not found, skip version check
+        return
+
     running = __version__
 
     if parse_version(installed) > parse_version(running):

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.8.28"
+__version__ = "2.10.0-rc.1"

validmind/ai/utils.py

@@ -3,9 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import os
-from urllib.parse import urljoin
 
-from openai import AzureOpenAI, Client, OpenAI
+from openai import AzureOpenAI, OpenAI
 
 from ..logging import get_logger
 from ..utils import md_to_html
@@ -83,28 +82,9 @@ def get_client_and_model():
         logger.debug(f"Using Azure OpenAI {__model} for generating descriptions")
 
     else:
-        try:
-            # TODO: fix circular import
-            from ..api_client import get_ai_key, get_api_host
-
-            response = get_ai_key()
-            __client = Client(
-                base_url=(
-                    # TODO: improve this to be a bit more dynamic
-                    "http://localhost:4000/genai"
-                    if "localhost" in get_api_host()
-                    else urljoin(get_api_host(), "/genai")
-                ),
-                api_key=response["key"],
-            )
-            __model = "gpt-4o"  # TODO: backend should tell us which model to use
-            logger.debug(f"Using ValidMind {__model} for generating descriptions")
-        except Exception as e:
-            logger.debug(f"Failed to get API key: {e}")
-            raise ValueError(
-                "OPENAI_API_KEY, AZURE_OPENAI_KEY must be set, or your account "
-                "must be setup to use ValidMind's LLM in order to use LLM features"
-            )
+        raise ValueError(
+            "OPENAI_API_KEY, AZURE_OPENAI_KEY must be setup to use LLM features"
+        )
 
     return __client, __model
 

validmind/api_client.py

@@ -22,7 +22,7 @@ from ipywidgets import HTML, Accordion
 
 from .client_config import client_config
 from .errors import MissingAPICredentialsError, MissingModelIdError, raise_api_error
-from .logging import get_logger, init_sentry, send_single_error
+from .logging import get_logger, init_sentry, log_api_operation, send_single_error
 from .utils import NumpyEncoder, is_html, md_to_html, run_async
 from .vm_models import Figure
 
@@ -85,7 +85,7 @@ def _get_session() -> aiohttp.ClientSession:
     if not __api_session or __api_session.closed:
         __api_session = aiohttp.ClientSession(
             headers=_get_api_headers(),
-            timeout=aiohttp.ClientTimeout(total=30),
+            timeout=aiohttp.ClientTimeout(total=int(os.getenv("VM_API_TIMEOUT", 30))),
         )
 
     return __api_session
@@ -304,6 +304,10 @@ async def alog_metadata(
         raise e
 
 
+@log_api_operation(
+    operation_name="Sending figure to ValidMind API",
+    extract_key=lambda figure: figure.key,
+)
 async def alog_figure(figure: Figure) -> Dict[str, Any]:
     """Logs a figure.
 
@@ -525,21 +529,6 @@ def log_metric(
     )
 
 
-def get_ai_key() -> Dict[str, Any]:
-    """Calls the API to get an API key for our LLM proxy."""
-    r = requests.get(
-        url=_get_url("ai/key"),
-        headers=_get_api_headers(),
-    )
-
-    if r.status_code != 200:
-        # TODO: improve error handling when there's no Open AI API or AI key available
-        # logger.error("Could not get AI key from ValidMind API")
-        raise_api_error(r.text)
-
-    return r.json()
-
-
 def generate_test_result_description(test_result_data: Dict[str, Any]) -> str:
     r = requests.post(
         url=_get_url("ai/generate/test_result_description"),

validmind/datasets/credit_risk/lending_club.py

@@ -9,13 +9,25 @@ from typing import Any, Dict, Optional, Tuple
 
 import numpy as np
 import pandas as pd
-import scorecardpy as sc
 import statsmodels.api as sm
 import xgboost as xgb
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.model_selection import train_test_split
 
 import validmind as vm
+from validmind.errors import MissingDependencyError
+
+try:
+    import scorecardpy as sc
+except ImportError as e:
+    if "scorecardpy" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `scorecardpy` for credit risk demos. "
+            "Please run `pip install validmind[credit_risk]` or `pip install scorecardpy`.",
+            required_dependencies=["scorecardpy"],
+            extra="credit_risk",
+        ) from e
+    raise e
 
 current_path = os.path.dirname(os.path.abspath(__file__))
 dataset_path = os.path.join(current_path, "datasets")

validmind/datasets/nlp/cnn_dailymail.py

@@ -7,10 +7,11 @@ import textwrap
 from typing import Optional, Tuple
 
 import pandas as pd
-from datasets import load_dataset
 from IPython.display import HTML, display
 from tabulate import tabulate
 
+from validmind.errors import MissingDependencyError
+
 # Define column names
 text_column = "article"
 target_column = "highlights"
@@ -37,6 +38,19 @@ def load_data(
         Tuple containing (train_df, test_df) DataFrames with the loaded data.
     """
     if source == "online":
+        try:
+            from datasets import load_dataset
+        except ImportError as e:
+            if "datasets" in str(e):
+                raise MissingDependencyError(
+                    "Missing required package `datasets` for CNN Daily Mail. "
+                    "Please run `pip install validmind[datasets]` or "
+                    "`pip install datasets` to use CNN Daily Mail dataset",
+                    required_dependencies=["datasets"],
+                    extra="datasets",
+                ) from e
+            raise e
+
         # Load online data without predictions
         cnn_dataset = load_dataset("cnn_dailymail", "3.0.0")
         train_df = cnn_dataset["train"].to_pandas()

validmind/logging.py

@@ -170,6 +170,54 @@ async def log_performance_async(
     return wrap
 
 
+def log_api_operation(
+    operation_name: Optional[str] = None,
+    logger: Optional[logging.Logger] = None,
+    extract_key: Optional[Callable] = None,
+    force: bool = False,
+) -> Callable[[F], F]:
+    """Decorator to log API operations like figure uploads.
+
+    Args:
+        operation_name (str, optional): The name of the operation. Defaults to function name.
+        logger (logging.Logger, optional): The logger to use. Defaults to None.
+        extract_key (Callable, optional): Function to extract a key from args for logging.
+        force (bool, optional): Whether to force logging even if env var is off.
+
+    Returns:
+        Callable: The decorated function.
+    """
+
+    def decorator(func: F) -> F:
+        # check if log level is set to debug
+        if _get_log_level() != logging.DEBUG and not force:
+            return func
+
+        nonlocal logger
+        if logger is None:
+            logger = get_logger()
+
+        nonlocal operation_name
+        if operation_name is None:
+            operation_name = func.__name__
+
+        async def wrapped(*args: Any, **kwargs: Any) -> Any:
+            # Try to extract a meaningful identifier from the arguments
+            identifier = ""
+            if extract_key and args:
+                try:
+                    identifier = f": {extract_key(args[0])}"
+                except (AttributeError, IndexError):
+                    pass
+
+            logger.debug(f"{operation_name}{identifier}")
+            return await func(*args, **kwargs)
+
+        return wrapped
+
+    return decorator
+
+
 def send_single_error(error: Exception) -> None:
     """Send a single error to Sentry.
 

validmind/tests/__init__.py

@@ -43,6 +43,8 @@ __all__ = [
     "data_validation",
     "model_validation",
     "prompt_validation",
+    "plots",
+    "stats",
     "list_tests",
     "load_test",
     "describe_test",

validmind/tests/__types__.py

@@ -187,6 +187,10 @@ TestID = Union[
         "validmind.ongoing_monitoring.ScoreBandsDrift",
         "validmind.ongoing_monitoring.ScorecardHistogramDrift",
         "validmind.ongoing_monitoring.TargetPredictionDistributionPlot",
+        "validmind.plots.BoxPlot",
+        "validmind.plots.CorrelationHeatmap",
+        "validmind.plots.HistogramPlot",
+        "validmind.plots.ViolinPlot",
         "validmind.prompt_validation.Bias",
         "validmind.prompt_validation.Clarity",
         "validmind.prompt_validation.Conciseness",
@@ -194,11 +198,25 @@ TestID = Union[
         "validmind.prompt_validation.NegativeInstruction",
         "validmind.prompt_validation.Robustness",
         "validmind.prompt_validation.Specificity",
+        "validmind.stats.CorrelationAnalysis",
+        "validmind.stats.DescriptiveStats",
+        "validmind.stats.NormalityTests",
+        "validmind.stats.OutlierDetection",
         "validmind.unit_metrics.classification.Accuracy",
         "validmind.unit_metrics.classification.F1",
         "validmind.unit_metrics.classification.Precision",
         "validmind.unit_metrics.classification.ROC_AUC",
         "validmind.unit_metrics.classification.Recall",
+        "validmind.unit_metrics.classification.individual.AbsoluteError",
+        "validmind.unit_metrics.classification.individual.BrierScore",
+        "validmind.unit_metrics.classification.individual.CalibrationError",
+        "validmind.unit_metrics.classification.individual.ClassBalance",
+        "validmind.unit_metrics.classification.individual.Confidence",
+        "validmind.unit_metrics.classification.individual.Correctness",
+        "validmind.unit_metrics.classification.individual.LogLoss",
+        "validmind.unit_metrics.classification.individual.OutlierScore",
+        "validmind.unit_metrics.classification.individual.ProbabilityError",
+        "validmind.unit_metrics.classification.individual.Uncertainty",
         "validmind.unit_metrics.regression.AdjustedRSquaredScore",
         "validmind.unit_metrics.regression.GiniCoefficient",
         "validmind.unit_metrics.regression.HuberLoss",

validmind/tests/data_validation/ChiSquaredFeaturesTable.py

@@ -4,10 +4,22 @@
 
 
 import pandas as pd
-from scipy.stats import chi2_contingency
 
 from validmind import tags, tasks
-from validmind.errors import SkipTestError
+from validmind.errors import MissingDependencyError, SkipTestError
+
+try:
+    from scipy.stats import chi2_contingency
+except ImportError as e:
+    if "scipy" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `scipy` for ChiSquaredFeaturesTable. "
+            "Please run `pip install validmind[stats]` to use statistical tests",
+            required_dependencies=["scipy"],
+            extra="stats",
+        ) from e
+
+    raise e
 
 
 @tags("tabular_data", "categorical_data", "statistical_test")

validmind/tests/data_validation/DickeyFullerGLS.py

@@ -5,14 +5,25 @@
 from typing import Any, Dict, Tuple
 
 import pandas as pd
-from arch.unitroot import DFGLS
 from numpy.linalg import LinAlgError
 
 from validmind import RawData, tags, tasks
-from validmind.errors import SkipTestError
+from validmind.errors import MissingDependencyError, SkipTestError
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset
 
+try:
+    from arch.unitroot import DFGLS
+except ImportError as e:
+    if "arch" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `arch` for DickeyFullerGLS. "
+            "Please run `pip install validmind[stats]` to use statistical tests",
+            required_dependencies=["arch"],
+            extra="stats",
+        ) from e
+    raise e
+
 logger = get_logger(__name__)
 
 

validmind/tests/data_validation/PhillipsPerronArch.py

@@ -6,14 +6,25 @@ from typing import Any, Dict
 
 import numpy as np
 import pandas as pd
-from arch.unitroot import PhillipsPerron
 from numpy.linalg import LinAlgError
 
 from validmind import tags, tasks
-from validmind.errors import SkipTestError
+from validmind.errors import MissingDependencyError, SkipTestError
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset
 
+try:
+    from arch.unitroot import PhillipsPerron
+except ImportError as e:
+    if "arch" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `arch` for PhillipsPerronArch. "
+            "Please run `pip install validmind[stats]` to use statistical tests",
+            required_dependencies=["arch"],
+            extra="stats",
+        ) from e
+    raise e
+
 logger = get_logger(__name__)
 
 

validmind/tests/data_validation/SeasonalDecompose.py

@@ -9,11 +9,23 @@ import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
-from scipy import stats
 from statsmodels.tsa.seasonal import seasonal_decompose
 
 from validmind import RawData, tags, tasks
-from validmind.errors import SkipTestError
+from validmind.errors import MissingDependencyError, SkipTestError
+
+try:
+    from scipy import stats
+except ImportError as e:
+    if "scipy" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `scipy` for SeasonalDecompose. "
+            "Please run `pip install validmind[stats]` to use statistical tests",
+            required_dependencies=["scipy"],
+            extra="stats",
+        ) from e
+
+    raise e
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset
 

validmind/tests/data_validation/ShapiroWilk.py

@@ -6,9 +6,22 @@
 from typing import Tuple
 
 import pandas as pd
-from scipy import stats
 
 from validmind import RawData, tags, tasks
+from validmind.errors import MissingDependencyError
+
+try:
+    from scipy import stats
+except ImportError as e:
+    if "scipy" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `scipy` for ShapiroWilk. "
+            "Please run `pip install validmind[stats]` to use statistical tests",
+            required_dependencies=["scipy"],
+            extra="stats",
+        ) from e
+
+    raise e
 
 
 @tasks("classification", "regression")

validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.py

@@ -3,9 +3,22 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import pandas as pd
-from scipy.stats import kurtosis, skew
 
 from validmind import tags, tasks
+from validmind.errors import MissingDependencyError
+
+try:
+    from scipy.stats import kurtosis, skew
+except ImportError as e:
+    if "scipy" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `scipy` for TimeSeriesDescriptiveStatistics. "
+            "Please run `pip install validmind[stats]` to use statistical tests",
+            required_dependencies=["scipy"],
+            extra="stats",
+        ) from e
+
+    raise e
 
 
 @tags("time_series_data", "analysis")

validmind/tests/data_validation/WOEBinPlots.py

@@ -9,7 +9,20 @@ import numpy as np
 import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
-import scorecardpy as sc
+
+from validmind.errors import MissingDependencyError
+
+try:
+    import scorecardpy as sc
+except ImportError as e:
+    if "scorecardpy" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `scorecardpy` for WOEBinPlots. "
+            "Please run `pip install validmind[credit_risk]` to use these tests",
+            required_dependencies=["scorecardpy"],
+            extra="credit_risk",
+        ) from e
+    raise e
 from plotly.subplots import make_subplots
 
 from validmind import RawData, tags, tasks

validmind/tests/data_validation/WOEBinTable.py

@@ -5,12 +5,23 @@
 from typing import Dict, Tuple
 
 import pandas as pd
-import scorecardpy as sc
 
 from validmind import RawData, tags, tasks
-from validmind.errors import SkipTestError
+from validmind.errors import MissingDependencyError, SkipTestError
 from validmind.vm_models import VMDataset
 
+try:
+    import scorecardpy as sc
+except ImportError as e:
+    if "scorecardpy" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `scorecardpy` for WOEBinTable. "
+            "Please run `pip install validmind[credit_risk]` to use these tests",
+            required_dependencies=["scorecardpy"],
+            extra="credit_risk",
+        ) from e
+    raise e
+
 
 @tags("tabular_data", "categorical_data")
 @tasks("classification")

validmind/tests/data_validation/ZivotAndrewsArch.py

@@ -5,14 +5,25 @@
 from typing import Any, Dict, Tuple
 
 import pandas as pd
-from arch.unitroot import ZivotAndrews
 from numpy.linalg import LinAlgError
 
 from validmind import RawData, tags, tasks
-from validmind.errors import SkipTestError
+from validmind.errors import MissingDependencyError, SkipTestError
 from validmind.logging import get_logger
 from validmind.vm_models import VMDataset
 
+try:
+    from arch.unitroot import ZivotAndrews
+except ImportError as e:
+    if "arch" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `arch` for ZivotAndrewsArch. "
+            "Please run `pip install validmind[stats]` to use statistical tests",
+            required_dependencies=["arch"],
+            extra="stats",
+        ) from e
+    raise e
+
 logger = get_logger(__name__)
 
 

validmind/tests/data_validation/nlp/CommonWords.py

@@ -5,13 +5,25 @@
 from collections import Counter
 from typing import Tuple
 
-import nltk
 import plotly.graph_objects as go
-from nltk.corpus import stopwords
 
 from validmind import RawData, tags, tasks
+from validmind.errors import MissingDependencyError
 from validmind.vm_models import VMDataset
 
+try:
+    import nltk
+    from nltk.corpus import stopwords
+except ImportError as e:
+    if "nltk" in str(e).lower():
+        raise MissingDependencyError(
+            "Missing required package `nltk` for CommonWords. "
+            "Please run `pip install validmind[nlp]` to use NLP tests",
+            required_dependencies=["nltk"],
+            extra="nlp",
+        ) from e
+    raise e
+
 
 @tags("nlp", "text_data", "visualization", "frequency_analysis")
 @tasks("text_classification", "text_summarization")

validmind/tests/data_validation/nlp/LanguageDetection.py

@@ -6,9 +6,22 @@ from typing import Tuple
 
 import plotly.express as px
 import plotly.graph_objects as go
-from langdetect import LangDetectException, detect
 
 from validmind import RawData, tags, tasks
+from validmind.errors import MissingDependencyError
+
+try:
+    from langdetect import LangDetectException, detect
+except ImportError as e:
+    if "langdetect" in str(e):
+        raise MissingDependencyError(
+            "Missing required package `langdetect` for LanguageDetection. "
+            "Please run `pip install validmind[nlp]` to use NLP tests",
+            required_dependencies=["langdetect"],
+            extra="nlp",
+        ) from e
+
+    raise e
 
 
 @tags("nlp", "text_data", "visualization")

validmind/tests/data_validation/nlp/PolarityAndSubjectivity.py

@@ -8,9 +8,21 @@ from typing import Dict, Tuple
 import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
-from textblob import TextBlob
 
 from validmind import RawData, tags, tasks
+from validmind.errors import MissingDependencyError
+
+try:
+    from textblob import TextBlob
+except ImportError as e:
+    if "textblob" in str(e).lower():
+        raise MissingDependencyError(
+            "Missing required package `textblob` for PolarityAndSubjectivity. "
+            "Please run `pip install validmind[nlp]` to use NLP tests",
+            required_dependencies=["textblob"],
+            extra="nlp",
+        ) from e
+    raise e
 
 
 @tags("nlp", "text_data", "data_validation")

validmind/tests/data_validation/nlp/Sentiment.py

@@ -8,9 +8,21 @@ from typing import Tuple
 import matplotlib.pyplot as plt
 import nltk
 import seaborn as sns
-from nltk.sentiment import SentimentIntensityAnalyzer
 
 from validmind import RawData, tags, tasks
+from validmind.errors import MissingDependencyError
+
+try:
+    from nltk.sentiment import SentimentIntensityAnalyzer
+except ImportError as e:
+    if "nltk" in str(e).lower():
+        raise MissingDependencyError(
+            "Missing required package `nltk` for Sentiment. "
+            "Please run `pip install validmind[nlp]` to use NLP tests",
+            required_dependencies=["nltk"],
+            extra="nlp",
+        ) from e
+    raise e
 
 
 @tags("nlp", "text_data", "data_validation")

validmind/tests/data_validation/nlp/StopWords.py

@@ -9,14 +9,26 @@ Threshold based tests
 from collections import defaultdict
 from typing import Dict, Tuple
 
-import nltk
 import pandas as pd
 import plotly.graph_objects as go
-from nltk.corpus import stopwords
 
 from validmind import RawData, tags, tasks
+from validmind.errors import MissingDependencyError
 from validmind.vm_models import VMDataset
 
+try:
+    import nltk
+    from nltk.corpus import stopwords
+except ImportError as e:
+    if "nltk" in str(e).lower():
+        raise MissingDependencyError(
+            "Missing required package `nltk` for StopWords. "
+            "Please run `pip install validmind[nlp]` to use NLP tests",
+            required_dependencies=["nltk"],
+            extra="nlp",
+        ) from e
+    raise e
+
 
 @tags("nlp", "text_data", "frequency_analysis", "visualization")
 @tasks("text_classification", "text_summarization")