validmind 2.8.29__py3-none-any.whl → 2.9.1__py3-none-any.whl

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.8.28"
+__version__ = "2.9.1"

validmind/ai/utils.py

@@ -3,9 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
 import os
-from urllib.parse import urljoin
 
-from openai import AzureOpenAI, Client, OpenAI
+from openai import AzureOpenAI, OpenAI
 
 from ..logging import get_logger
 from ..utils import md_to_html
@@ -83,28 +82,9 @@ def get_client_and_model():
         logger.debug(f"Using Azure OpenAI {__model} for generating descriptions")
 
     else:
-        try:
-            # TODO: fix circular import
-            from ..api_client import get_ai_key, get_api_host
-
-            response = get_ai_key()
-            __client = Client(
-                base_url=(
-                    # TODO: improve this to be a bit more dynamic
-                    "http://localhost:4000/genai"
-                    if "localhost" in get_api_host()
-                    else urljoin(get_api_host(), "/genai")
-                ),
-                api_key=response["key"],
-            )
-            __model = "gpt-4o"  # TODO: backend should tell us which model to use
-            logger.debug(f"Using ValidMind {__model} for generating descriptions")
-        except Exception as e:
-            logger.debug(f"Failed to get API key: {e}")
-            raise ValueError(
-                "OPENAI_API_KEY, AZURE_OPENAI_KEY must be set, or your account "
-                "must be setup to use ValidMind's LLM in order to use LLM features"
-            )
+        raise ValueError(
+            "OPENAI_API_KEY, AZURE_OPENAI_KEY must be setup to use LLM features"
+        )
 
     return __client, __model
 

validmind/api_client.py

@@ -22,7 +22,7 @@ from ipywidgets import HTML, Accordion
 
 from .client_config import client_config
 from .errors import MissingAPICredentialsError, MissingModelIdError, raise_api_error
-from .logging import get_logger, init_sentry, send_single_error
+from .logging import get_logger, init_sentry, log_api_operation, send_single_error
 from .utils import NumpyEncoder, is_html, md_to_html, run_async
 from .vm_models import Figure
 
@@ -85,7 +85,7 @@ def _get_session() -> aiohttp.ClientSession:
     if not __api_session or __api_session.closed:
         __api_session = aiohttp.ClientSession(
             headers=_get_api_headers(),
-            timeout=aiohttp.ClientTimeout(total=30),
+            timeout=aiohttp.ClientTimeout(total=int(os.getenv("VM_API_TIMEOUT", 30))),
         )
 
     return __api_session
@@ -304,6 +304,10 @@ async def alog_metadata(
         raise e
 
 
+@log_api_operation(
+    operation_name="Sending figure to ValidMind API",
+    extract_key=lambda figure: figure.key,
+)
 async def alog_figure(figure: Figure) -> Dict[str, Any]:
     """Logs a figure.
 
@@ -525,21 +529,6 @@ def log_metric(
     )
 
 
-def get_ai_key() -> Dict[str, Any]:
-    """Calls the API to get an API key for our LLM proxy."""
-    r = requests.get(
-        url=_get_url("ai/key"),
-        headers=_get_api_headers(),
-    )
-
-    if r.status_code != 200:
-        # TODO: improve error handling when there's no Open AI API or AI key available
-        # logger.error("Could not get AI key from ValidMind API")
-        raise_api_error(r.text)
-
-    return r.json()
-
-
 def generate_test_result_description(test_result_data: Dict[str, Any]) -> str:
     r = requests.post(
         url=_get_url("ai/generate/test_result_description"),

validmind/logging.py

@@ -170,6 +170,54 @@ async def log_performance_async(
     return wrap
 
 
+def log_api_operation(
+    operation_name: Optional[str] = None,
+    logger: Optional[logging.Logger] = None,
+    extract_key: Optional[Callable] = None,
+    force: bool = False,
+) -> Callable[[F], F]:
+    """Decorator to log API operations like figure uploads.
+
+    Args:
+        operation_name (str, optional): The name of the operation. Defaults to function name.
+        logger (logging.Logger, optional): The logger to use. Defaults to None.
+        extract_key (Callable, optional): Function to extract a key from args for logging.
+        force (bool, optional): Whether to force logging even if env var is off.
+
+    Returns:
+        Callable: The decorated function.
+    """
+
+    def decorator(func: F) -> F:
+        # check if log level is set to debug
+        if _get_log_level() != logging.DEBUG and not force:
+            return func
+
+        nonlocal logger
+        if logger is None:
+            logger = get_logger()
+
+        nonlocal operation_name
+        if operation_name is None:
+            operation_name = func.__name__
+
+        async def wrapped(*args: Any, **kwargs: Any) -> Any:
+            # Try to extract a meaningful identifier from the arguments
+            identifier = ""
+            if extract_key and args:
+                try:
+                    identifier = f": {extract_key(args[0])}"
+                except (AttributeError, IndexError):
+                    pass
+
+            logger.debug(f"{operation_name}{identifier}")
+            return await func(*args, **kwargs)
+
+        return wrapped
+
+    return decorator
+
+
 def send_single_error(error: Exception) -> None:
     """Send a single error to Sentry.
 

validmind/tests/__init__.py

@@ -43,6 +43,8 @@ __all__ = [
     "data_validation",
     "model_validation",
     "prompt_validation",
+    "plots",
+    "stats",
     "list_tests",
     "load_test",
     "describe_test",

validmind/tests/__types__.py

@@ -187,6 +187,10 @@ TestID = Union[
         "validmind.ongoing_monitoring.ScoreBandsDrift",
         "validmind.ongoing_monitoring.ScorecardHistogramDrift",
         "validmind.ongoing_monitoring.TargetPredictionDistributionPlot",
+        "validmind.plots.BoxPlot",
+        "validmind.plots.CorrelationHeatmap",
+        "validmind.plots.HistogramPlot",
+        "validmind.plots.ViolinPlot",
         "validmind.prompt_validation.Bias",
         "validmind.prompt_validation.Clarity",
         "validmind.prompt_validation.Conciseness",
@@ -194,11 +198,25 @@ TestID = Union[
         "validmind.prompt_validation.NegativeInstruction",
         "validmind.prompt_validation.Robustness",
         "validmind.prompt_validation.Specificity",
+        "validmind.stats.CorrelationAnalysis",
+        "validmind.stats.DescriptiveStats",
+        "validmind.stats.NormalityTests",
+        "validmind.stats.OutlierDetection",
         "validmind.unit_metrics.classification.Accuracy",
         "validmind.unit_metrics.classification.F1",
         "validmind.unit_metrics.classification.Precision",
         "validmind.unit_metrics.classification.ROC_AUC",
         "validmind.unit_metrics.classification.Recall",
+        "validmind.unit_metrics.classification.individual.AbsoluteError",
+        "validmind.unit_metrics.classification.individual.BrierScore",
+        "validmind.unit_metrics.classification.individual.CalibrationError",
+        "validmind.unit_metrics.classification.individual.ClassBalance",
+        "validmind.unit_metrics.classification.individual.Confidence",
+        "validmind.unit_metrics.classification.individual.Correctness",
+        "validmind.unit_metrics.classification.individual.LogLoss",
+        "validmind.unit_metrics.classification.individual.OutlierScore",
+        "validmind.unit_metrics.classification.individual.ProbabilityError",
+        "validmind.unit_metrics.classification.individual.Uncertainty",
         "validmind.unit_metrics.regression.AdjustedRSquaredScore",
         "validmind.unit_metrics.regression.GiniCoefficient",
         "validmind.unit_metrics.regression.HuberLoss",

validmind/tests/output.py

@@ -45,7 +45,13 @@ class BooleanOutputHandler(OutputHandler):
 
 class MetricOutputHandler(OutputHandler):
     def can_handle(self, item: Any) -> bool:
-        return isinstance(item, (int, float))
+        # Accept individual numbers
+        if isinstance(item, (int, float)):
+            return True
+        # Accept lists/arrays of numbers for per-row metrics
+        if isinstance(item, (list, tuple, np.ndarray)):
+            return all(isinstance(x, (int, float, np.number)) for x in item)
+        return False
 
     def process(self, item: Any, result: TestResult) -> None:
         if result.metric is not None:
@@ -169,11 +175,12 @@ def process_output(item: Any, result: TestResult) -> None:
     """Process a single test output item and update the TestResult."""
     handlers = [
         BooleanOutputHandler(),
-        MetricOutputHandler(),
         FigureOutputHandler(),
         TableOutputHandler(),
         RawDataOutputHandler(),
         StringOutputHandler(),
+        # Unit metrics should be processed last
+        MetricOutputHandler(),
     ]
 
     for handler in handlers:

validmind/tests/plots/BoxPlot.py

@@ -0,0 +1,260 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List, Optional
+
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.vm_models import VMDataset
+
+
+def _validate_inputs(
+    dataset: VMDataset, columns: Optional[List[str]], group_by: Optional[str]
+):
+    """Validate inputs and return validated columns."""
+    if columns is None:
+        columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        columns = [col for col in columns if col in available_columns]
+
+    if not columns:
+        raise SkipTestError("No numerical columns found for box plotting")
+
+    if group_by is not None:
+        if group_by not in dataset.df.columns:
+            raise SkipTestError(f"Group column '{group_by}' not found in dataset")
+        if group_by in columns:
+            columns.remove(group_by)
+
+    return columns
+
+
+def _create_grouped_boxplot(
+    dataset, columns, group_by, colors, show_outliers, title_prefix, width, height
+):
+    """Create grouped box plots."""
+    fig = go.Figure()
+    groups = dataset.df[group_by].dropna().unique()
+
+    for col_idx, column in enumerate(columns):
+        for group_idx, group_value in enumerate(groups):
+            data_subset = dataset.df[dataset.df[group_by] == group_value][
+                column
+            ].dropna()
+
+            if len(data_subset) > 0:
+                color = colors[group_idx % len(colors)]
+                fig.add_trace(
+                    go.Box(
+                        y=data_subset,
+                        name=f"{group_value}",
+                        marker_color=color,
+                        boxpoints="outliers" if show_outliers else False,
+                        jitter=0.3,
+                        pointpos=-1.8,
+                        legendgroup=f"{group_value}",
+                        showlegend=(col_idx == 0),
+                        offsetgroup=group_idx,
+                        x=[column] * len(data_subset),
+                    )
+                )
+
+    fig.update_layout(
+        title=f"{title_prefix} Features by {group_by}",
+        xaxis_title="Features",
+        yaxis_title="Values",
+        boxmode="group",
+        width=width,
+        height=height,
+        template="plotly_white",
+    )
+    return fig
+
+
+def _create_single_boxplot(
+    dataset, column, colors, show_outliers, title_prefix, width, height
+):
+    """Create single column box plot."""
+    data = dataset.df[column].dropna()
+    if len(data) == 0:
+        raise SkipTestError(f"No data available for column {column}")
+
+    fig = go.Figure()
+    fig.add_trace(
+        go.Box(
+            y=data,
+            name=column,
+            marker_color=colors[0],
+            boxpoints="outliers" if show_outliers else False,
+            jitter=0.3,
+            pointpos=-1.8,
+        )
+    )
+
+    fig.update_layout(
+        title=f"{title_prefix} {column}",
+        yaxis_title=column,
+        width=width,
+        height=height,
+        template="plotly_white",
+        showlegend=False,
+    )
+    return fig
+
+
+def _create_multiple_boxplots(
+    dataset, columns, colors, show_outliers, title_prefix, width, height
+):
+    """Create multiple column box plots in subplot layout."""
+    n_cols = min(3, len(columns))
+    n_rows = (len(columns) + n_cols - 1) // n_cols
+
+    subplot_titles = [f"{title_prefix} {col}" for col in columns]
+    fig = make_subplots(
+        rows=n_rows,
+        cols=n_cols,
+        subplot_titles=subplot_titles,
+        vertical_spacing=0.1,
+        horizontal_spacing=0.1,
+    )
+
+    for idx, column in enumerate(columns):
+        row = (idx // n_cols) + 1
+        col = (idx % n_cols) + 1
+        data = dataset.df[column].dropna()
+
+        if len(data) > 0:
+            color = colors[idx % len(colors)]
+            fig.add_trace(
+                go.Box(
+                    y=data,
+                    name=column,
+                    marker_color=color,
+                    boxpoints="outliers" if show_outliers else False,
+                    jitter=0.3,
+                    pointpos=-1.8,
+                    showlegend=False,
+                ),
+                row=row,
+                col=col,
+            )
+            fig.update_yaxes(title_text=column, row=row, col=col)
+        else:
+            fig.add_annotation(
+                text=f"No data available<br>for {column}",
+                x=0.5,
+                y=0.5,
+                xref=f"x{idx+1} domain" if idx > 0 else "x domain",
+                yref=f"y{idx+1} domain" if idx > 0 else "y domain",
+                showarrow=False,
+                row=row,
+                col=col,
+            )
+
+    fig.update_layout(
+        title="Dataset Feature Distributions",
+        width=width,
+        height=height,
+        template="plotly_white",
+        showlegend=False,
+    )
+    return fig
+
+
+@tags("tabular_data", "visualization", "data_quality")
+@tasks("classification", "regression", "clustering")
+def BoxPlot(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    group_by: Optional[str] = None,
+    width: int = 1200,
+    height: int = 600,
+    colors: Optional[List[str]] = None,
+    show_outliers: bool = True,
+    title_prefix: str = "Box Plot of",
+) -> go.Figure:
+    """
+    Generates customizable box plots for numerical features in a dataset with optional grouping using Plotly.
+
+    ### Purpose
+
+    This test provides a flexible way to visualize the distribution of numerical features
+    through interactive box plots, with optional grouping by categorical variables. Box plots are
+    effective for identifying outliers, comparing distributions across groups, and
+    understanding the spread and central tendency of the data.
+
+    ### Test Mechanism
+
+    The test creates interactive box plots for specified numerical columns (or all numerical columns
+    if none specified). It supports various customization options including:
+    - Grouping by categorical variables
+    - Customizable colors and styling
+    - Outlier display options
+    - Interactive hover information
+    - Zoom and pan capabilities
+
+    ### Signs of High Risk
+
+    - Presence of many outliers indicating data quality issues
+    - Highly skewed distributions
+    - Large differences in variance across groups
+    - Unexpected patterns in grouped data
+
+    ### Strengths
+
+    - Clear visualization of distribution statistics (median, quartiles, outliers)
+    - Interactive Plotly plots with hover information and zoom capabilities
+    - Effective for comparing distributions across groups
+    - Handles missing values appropriately
+    - Highly customizable appearance
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - May not be suitable for continuous variables with many unique values
+    - Visual interpretation may be subjective
+    - Less effective with very large datasets
+    """
+    # Validate inputs
+    columns = _validate_inputs(dataset, columns, group_by)
+
+    # Set default colors
+    if colors is None:
+        colors = [
+            "steelblue",
+            "orange",
+            "green",
+            "red",
+            "purple",
+            "brown",
+            "pink",
+            "gray",
+            "olive",
+            "cyan",
+        ]
+
+    # Create appropriate plot type
+    if group_by is not None:
+        return _create_grouped_boxplot(
+            dataset,
+            columns,
+            group_by,
+            colors,
+            show_outliers,
+            title_prefix,
+            width,
+            height,
+        )
+    elif len(columns) == 1:
+        return _create_single_boxplot(
+            dataset, columns[0], colors, show_outliers, title_prefix, width, height
+        )
+    else:
+        return _create_multiple_boxplots(
+            dataset, columns, colors, show_outliers, title_prefix, width, height
+        )