validmind 2.8.28__py3-none-any.whl → 2.9.1__py3-none-any.whl

validmind/tests/plots/CorrelationHeatmap.py

@@ -0,0 +1,235 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List, Optional
+
+import numpy as np
+import plotly.graph_objects as go
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.vm_models import VMDataset
+
+
+def _validate_and_prepare_data(
+    dataset: VMDataset, columns: Optional[List[str]], method: str
+):
+    """Validate inputs and prepare correlation data."""
+    if columns is None:
+        columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        columns = [col for col in columns if col in available_columns]
+
+    if not columns:
+        raise SkipTestError("No numerical columns found for correlation analysis")
+
+    if len(columns) < 2:
+        raise SkipTestError(
+            "At least 2 numerical columns required for correlation analysis"
+        )
+
+    # Get data and remove constant columns
+    data = dataset.df[columns]
+    data = data.loc[:, data.var() != 0]
+
+    if data.shape[1] < 2:
+        raise SkipTestError(
+            "Insufficient non-constant columns for correlation analysis"
+        )
+
+    return data.corr(method=method)
+
+
+def _apply_filters(corr_matrix, threshold: Optional[float], mask_upper: bool):
+    """Apply threshold and masking filters to correlation matrix."""
+    if threshold is not None:
+        mask = np.abs(corr_matrix) < threshold
+        corr_matrix = corr_matrix.mask(mask)
+
+    if mask_upper:
+        mask = np.triu(np.ones_like(corr_matrix, dtype=bool))
+        corr_matrix = corr_matrix.mask(mask)
+
+    return corr_matrix
+
+
+def _create_annotation_text(z_values, y_labels, x_labels, show_values: bool):
+    """Create text annotations for heatmap cells."""
+    if not show_values:
+        return None
+
+    text = []
+    for i in range(len(y_labels)):
+        text_row = []
+        for j in range(len(x_labels)):
+            value = z_values[i][j]
+            if np.isnan(value):
+                text_row.append("")
+            else:
+                text_row.append(f"{value:.3f}")
+        text.append(text_row)
+    return text
+
+
+def _calculate_adaptive_font_size(n_features: int) -> int:
+    """Calculate adaptive font size based on number of features."""
+    if n_features <= 10:
+        return 12
+    elif n_features <= 20:
+        return 10
+    elif n_features <= 30:
+        return 8
+    else:
+        return 6
+
+
+def _calculate_stats_and_update_layout(
+    fig, corr_matrix, method: str, title: str, width: int, height: int
+):
+    """Calculate statistics and update figure layout."""
+    n_features = corr_matrix.shape[0]
+    upper_triangle = corr_matrix.values[np.triu_indices_from(corr_matrix.values, k=1)]
+    upper_triangle = upper_triangle[~np.isnan(upper_triangle)]
+
+    if len(upper_triangle) > 0:
+        mean_corr = np.abs(upper_triangle).mean()
+        max_corr = np.abs(upper_triangle).max()
+        stats_text = f"Features: {n_features}<br>Mean |r|: {mean_corr:.3f}<br>Max |r|: {max_corr:.3f}"
+    else:
+        stats_text = f"Features: {n_features}"
+
+    fig.update_layout(
+        title={
+            "text": f"{title} ({method.capitalize()} Correlation)",
+            "x": 0.5,
+            "xanchor": "center",
+        },
+        width=width,
+        height=height,
+        template="plotly_white",
+        xaxis=dict(tickangle=45, side="bottom"),
+        yaxis=dict(tickmode="linear", autorange="reversed"),
+        annotations=[
+            dict(
+                text=stats_text,
+                x=0.02,
+                y=0.98,
+                xref="paper",
+                yref="paper",
+                showarrow=False,
+                align="left",
+                bgcolor="rgba(255,255,255,0.8)",
+                bordercolor="black",
+                borderwidth=1,
+            )
+        ],
+    )
+
+
+@tags("tabular_data", "visualization", "correlation")
+@tasks("classification", "regression", "clustering")
+def CorrelationHeatmap(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    method: str = "pearson",
+    show_values: bool = True,
+    colorscale: str = "RdBu",
+    width: int = 800,
+    height: int = 600,
+    mask_upper: bool = False,
+    threshold: Optional[float] = None,
+    title: str = "Correlation Heatmap",
+) -> go.Figure:
+    """
+    Generates customizable correlation heatmap plots for numerical features in a dataset using Plotly.
+
+    ### Purpose
+
+    This test provides a flexible way to visualize correlations between numerical features
+    in a dataset using interactive Plotly heatmaps. It supports different correlation methods
+    and extensive customization options for the heatmap appearance, making it suitable for
+    exploring feature relationships in data analysis.
+
+    ### Test Mechanism
+
+    The test computes correlation coefficients between specified numerical columns
+    (or all numerical columns if none specified) using the specified method.
+    It then creates an interactive heatmap visualization with customizable appearance options including:
+    - Different correlation methods (pearson, spearman, kendall)
+    - Color schemes and annotations
+    - Masking options for upper triangle
+    - Threshold filtering for significant correlations
+    - Interactive hover information
+
+    ### Signs of High Risk
+
+    - Very high correlations (>0.9) between features indicating multicollinearity
+    - Unexpected correlation patterns that contradict domain knowledge
+    - Features with no correlation to any other variables
+    - Strong correlations with the target variable that might indicate data leakage
+
+    ### Strengths
+
+    - Supports multiple correlation methods
+    - Interactive Plotly plots with hover information and zoom capabilities
+    - Highly customizable visualization options
+    - Can handle missing values appropriately
+    - Provides clear visual representation of feature relationships
+    - Optional thresholding to focus on significant correlations
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Cannot capture non-linear relationships effectively
+    - May be difficult to interpret with many features
+    - Correlation does not imply causation
+    """
+    # Validate inputs and compute correlation
+    corr_matrix = _validate_and_prepare_data(dataset, columns, method)
+
+    # Apply filters
+    corr_matrix = _apply_filters(corr_matrix, threshold, mask_upper)
+
+    # Prepare heatmap data
+    z_values = corr_matrix.values
+    x_labels = corr_matrix.columns.tolist()
+    y_labels = corr_matrix.index.tolist()
+    text = _create_annotation_text(z_values, y_labels, x_labels, show_values)
+
+    # Calculate adaptive font size
+    n_features = len(x_labels)
+    font_size = _calculate_adaptive_font_size(n_features)
+
+    # Create heatmap
+    heatmap_kwargs = {
+        "z": z_values,
+        "x": x_labels,
+        "y": y_labels,
+        "colorscale": colorscale,
+        "zmin": -1,
+        "zmax": 1,
+        "colorbar": dict(title=f"{method.capitalize()} Correlation"),
+        "hoverongaps": False,
+        "hovertemplate": "<b>%{y}</b> vs <b>%{x}</b><br>"
+        + f"{method.capitalize()} Correlation: %{{z:.3f}}<br>"
+        + "<extra></extra>",
+    }
+
+    # Add text annotations if requested
+    if show_values and text is not None:
+        heatmap_kwargs.update(
+            {
+                "text": text,
+                "texttemplate": "%{text}",
+                "textfont": {"size": font_size, "color": "black"},
+            }
+        )
+
+    fig = go.Figure(data=go.Heatmap(**heatmap_kwargs))
+
+    # Update layout with stats
+    _calculate_stats_and_update_layout(fig, corr_matrix, method, title, width, height)
+
+    return fig

validmind/tests/plots/HistogramPlot.py

@@ -0,0 +1,233 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List, Optional, Union
+
+import numpy as np
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+from scipy import stats
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.vm_models import VMDataset
+
+
+def _validate_columns(dataset: VMDataset, columns: Optional[List[str]]):
+    """Validate and return numerical columns."""
+    if columns is None:
+        columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        columns = [col for col in columns if col in available_columns]
+
+    if not columns:
+        raise SkipTestError("No numerical columns found for histogram plotting")
+
+    return columns
+
+
+def _process_column_data(data, log_scale: bool, column: str):
+    """Process column data and return plot data and xlabel."""
+    plot_data = data
+    xlabel = column
+    if log_scale and (data > 0).all():
+        plot_data = np.log10(data)
+        xlabel = f"log10({column})"
+    return plot_data, xlabel
+
+
+def _add_histogram_trace(
+    fig, plot_data, bins, color, opacity, normalize, column, row, col
+):
+    """Add histogram trace to figure."""
+    histnorm = "probability density" if normalize else None
+
+    fig.add_trace(
+        go.Histogram(
+            x=plot_data,
+            nbinsx=bins if isinstance(bins, int) else None,
+            name=f"Histogram - {column}",
+            marker_color=color,
+            opacity=opacity,
+            histnorm=histnorm,
+            showlegend=False,
+        ),
+        row=row,
+        col=col,
+    )
+
+
+def _add_kde_trace(fig, plot_data, bins, normalize, column, row, col):
+    """Add KDE trace to figure if possible."""
+    try:
+        kde = stats.gaussian_kde(plot_data)
+        x_range = np.linspace(plot_data.min(), plot_data.max(), 100)
+        kde_values = kde(x_range)
+
+        if not normalize:
+            hist_max = (
+                len(plot_data) / bins if isinstance(bins, int) else len(plot_data) / 30
+            )
+            kde_values = kde_values * hist_max / kde_values.max()
+
+        fig.add_trace(
+            go.Scatter(
+                x=x_range,
+                y=kde_values,
+                mode="lines",
+                name=f"KDE - {column}",
+                line=dict(color="red", width=2),
+                showlegend=False,
+            ),
+            row=row,
+            col=col,
+        )
+    except Exception:
+        pass
+
+
+def _add_stats_annotation(fig, data, idx, row, col):
+    """Add statistics annotation to subplot."""
+    stats_text = f"Mean: {data.mean():.3f}<br>Std: {data.std():.3f}<br>N: {len(data)}"
+    fig.add_annotation(
+        text=stats_text,
+        x=0.02,
+        y=0.98,
+        xref=f"x{idx+1} domain" if idx > 0 else "x domain",
+        yref=f"y{idx+1} domain" if idx > 0 else "y domain",
+        showarrow=False,
+        align="left",
+        bgcolor="rgba(255,255,255,0.8)",
+        bordercolor="black",
+        borderwidth=1,
+        row=row,
+        col=col,
+    )
+
+
+@tags("tabular_data", "visualization", "data_quality")
+@tasks("classification", "regression", "clustering")
+def HistogramPlot(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    bins: Union[int, str, List] = 30,
+    color: str = "steelblue",
+    opacity: float = 0.7,
+    show_kde: bool = True,
+    normalize: bool = False,
+    log_scale: bool = False,
+    title_prefix: str = "Histogram of",
+    width: int = 1200,
+    height: int = 800,
+    n_cols: int = 2,
+    vertical_spacing: float = 0.15,
+    horizontal_spacing: float = 0.1,
+) -> go.Figure:
+    """
+    Generates customizable histogram plots for numerical features in a dataset using Plotly.
+
+    ### Purpose
+
+    This test provides a flexible way to visualize the distribution of numerical features in a dataset.
+    It allows for extensive customization of the histogram appearance and behavior through parameters,
+    making it suitable for various exploratory data analysis tasks.
+
+    ### Test Mechanism
+
+    The test creates histogram plots for specified numerical columns (or all numerical columns if none specified).
+    It supports various customization options including:
+    - Number of bins or bin edges
+    - Color and opacity
+    - Kernel density estimation overlay
+    - Logarithmic scaling
+    - Normalization options
+    - Configurable subplot layout (columns and spacing)
+
+    ### Signs of High Risk
+
+    - Highly skewed distributions that may indicate data quality issues
+    - Unexpected bimodal or multimodal distributions
+    - Presence of extreme outliers
+    - Empty or sparse distributions
+
+    ### Strengths
+
+    - Highly customizable visualization options
+    - Interactive Plotly plots with zoom, pan, and hover capabilities
+    - Supports both single and multiple column analysis
+    - Provides insights into data distribution patterns
+    - Can handle different data types and scales
+    - Configurable subplot layout for better visualization
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Visual interpretation may be subjective
+    - May not be suitable for high-dimensional datasets
+    - Performance may degrade with very large datasets
+    """
+    # Validate inputs
+    columns = _validate_columns(dataset, columns)
+
+    # Calculate subplot layout
+    n_cols = min(n_cols, len(columns))
+    n_rows = (len(columns) + n_cols - 1) // n_cols
+
+    # Create subplots
+    subplot_titles = [f"{title_prefix} {col}" for col in columns]
+    fig = make_subplots(
+        rows=n_rows,
+        cols=n_cols,
+        subplot_titles=subplot_titles,
+        vertical_spacing=vertical_spacing,
+        horizontal_spacing=horizontal_spacing,
+    )
+
+    for idx, column in enumerate(columns):
+        row = (idx // n_cols) + 1
+        col = (idx % n_cols) + 1
+        data = dataset.df[column].dropna()
+
+        if len(data) == 0:
+            fig.add_annotation(
+                text=f"No data available<br>for {column}",
+                x=0.5,
+                y=0.5,
+                xref=f"x{idx+1}" if idx > 0 else "x",
+                yref=f"y{idx+1}" if idx > 0 else "y",
+                showarrow=False,
+                row=row,
+                col=col,
+            )
+            continue
+
+        # Process data
+        plot_data, xlabel = _process_column_data(data, log_scale, column)
+
+        # Add histogram
+        _add_histogram_trace(
+            fig, plot_data, bins, color, opacity, normalize, column, row, col
+        )
+
+        # Add KDE if requested
+        if show_kde and len(data) > 1:
+            _add_kde_trace(fig, plot_data, bins, normalize, column, row, col)
+
+        # Update axes and add annotations
+        fig.update_xaxes(title_text=xlabel, row=row, col=col)
+        ylabel = "Density" if normalize else "Frequency"
+        fig.update_yaxes(title_text=ylabel, row=row, col=col)
+        _add_stats_annotation(fig, data, idx, row, col)
+
+    # Update layout
+    fig.update_layout(
+        title_text="Dataset Feature Distributions",
+        showlegend=False,
+        width=width,
+        height=height,
+        template="plotly_white",
+    )
+
+    return fig

validmind/tests/plots/ViolinPlot.py

@@ -0,0 +1,125 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List, Optional
+
+import plotly.express as px
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.vm_models import VMDataset
+
+
+@tags("tabular_data", "visualization", "distribution")
+@tasks("classification", "regression", "clustering")
+def ViolinPlot(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    group_by: Optional[str] = None,
+    width: int = 800,
+    height: int = 600,
+) -> px.violin:
+    """
+    Generates interactive violin plots for numerical features using Plotly.
+
+    ### Purpose
+
+    This test creates violin plots to visualize the distribution of numerical features,
+    showing both the probability density and summary statistics. Violin plots combine
+    aspects of box plots and kernel density estimation for rich distribution visualization.
+
+    ### Test Mechanism
+
+    The test creates violin plots for specified numerical columns, with optional
+    grouping by categorical variables. Each violin shows the distribution shape,
+    quartiles, and median values.
+
+    ### Signs of High Risk
+
+    - Multimodal distributions that might indicate mixed populations
+    - Highly skewed distributions suggesting data quality issues
+    - Large differences in distribution shapes across groups
+    - Unusual distribution patterns that contradict domain expectations
+
+    ### Strengths
+
+    - Shows detailed distribution shape information
+    - Interactive Plotly visualization with hover details
+    - Effective for comparing distributions across groups
+    - Combines density estimation with quartile information
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Requires sufficient data points for meaningful density estimation
+    - May not be suitable for discrete variables
+    - Can be misleading with very small sample sizes
+    """
+    # Get numerical columns
+    if columns is None:
+        columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        columns = [col for col in columns if col in available_columns]
+
+    if not columns:
+        raise SkipTestError("No numerical columns found for violin plot")
+
+    # For violin plots, we'll melt the data to long format
+    data = dataset.df[columns].dropna()
+
+    if len(data) == 0:
+        raise SkipTestError("No valid data available for violin plot")
+
+    # Melt the dataframe to long format
+    melted_data = data.melt(var_name="Feature", value_name="Value")
+
+    # Add group column if specified
+    if group_by and group_by in dataset.df.columns:
+        # Repeat group values for each feature
+        group_values = []
+        for column in columns:
+            column_data = dataset.df[[column, group_by]].dropna()
+            group_values.extend(column_data[group_by].tolist())
+
+        if len(group_values) == len(melted_data):
+            melted_data["Group"] = group_values
+        else:
+            group_by = None  # Disable grouping if lengths don't match
+
+    # Create violin plot
+    if group_by and "Group" in melted_data.columns:
+        fig = px.violin(
+            melted_data,
+            x="Feature",
+            y="Value",
+            color="Group",
+            box=True,
+            title=f"Distribution of Features by {group_by}",
+            width=width,
+            height=height,
+        )
+    else:
+        fig = px.violin(
+            melted_data,
+            x="Feature",
+            y="Value",
+            box=True,
+            title="Feature Distributions",
+            width=width,
+            height=height,
+        )
+
+    # Update layout
+    fig.update_layout(
+        template="plotly_white",
+        title_x=0.5,
+        xaxis_title="Features",
+        yaxis_title="Values",
+    )
+
+    # Rotate x-axis labels for better readability
+    fig.update_xaxes(tickangle=45)
+
+    return fig

validmind/tests/prompt_validation/Bias.py

@@ -2,6 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Any, Dict, List, Tuple
+
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingRequiredTestInputError
 
@@ -45,7 +47,9 @@ Prompt:
 
 @tags("llm", "few_shot")
 @tasks("text_classification", "text_summarization")
-def Bias(model, min_threshold=7, judge_llm=None):
+def Bias(
+    model, min_threshold=7, judge_llm=None
+) -> Tuple[List[Dict[str, Any]], bool, RawData]:
     """
     Assesses potential bias in a Large Language Model by analyzing the distribution and order of exemplars in the
     prompt.

validmind/tests/prompt_validation/Clarity.py

@@ -2,6 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Any, Dict, List, Tuple
+
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingRequiredTestInputError
 
@@ -46,7 +48,9 @@ Prompt:
 
 @tags("llm", "zero_shot", "few_shot")
 @tasks("text_classification", "text_summarization")
-def Clarity(model, min_threshold=7, judge_llm=None):
+def Clarity(
+    model, min_threshold=7, judge_llm=None
+) -> Tuple[List[Dict[str, Any]], bool, RawData]:
     """
     Evaluates and scores the clarity of prompts in a Large Language Model based on specified guidelines.
 

validmind/tests/prompt_validation/Conciseness.py

@@ -2,6 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Any, Dict, List, Tuple
+
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingRequiredTestInputError
 
@@ -54,7 +56,9 @@ Prompt:
 
 @tags("llm", "zero_shot", "few_shot")
 @tasks("text_classification", "text_summarization")
-def Conciseness(model, min_threshold=7, judge_llm=None):
+def Conciseness(
+    model, min_threshold=7, judge_llm=None
+) -> Tuple[List[Dict[str, Any]], bool, RawData]:
     """
     Analyzes and grades the conciseness of prompts provided to a Large Language Model.
 

validmind/tests/prompt_validation/Delimitation.py

@@ -2,6 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Any, Dict, List, Tuple
+
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingRequiredTestInputError
 
@@ -39,7 +41,9 @@ Prompt:
 
 @tags("llm", "zero_shot", "few_shot")
 @tasks("text_classification", "text_summarization")
-def Delimitation(model, min_threshold=7, judge_llm=None):
+def Delimitation(
+    model, min_threshold=7, judge_llm=None
+) -> Tuple[List[Dict[str, Any]], bool, RawData]:
     """
     Evaluates the proper use of delimiters in prompts provided to Large Language Models.
 

validmind/tests/prompt_validation/NegativeInstruction.py

@@ -2,6 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+from typing import Any, Dict, List, Tuple
+
 from validmind import RawData, tags, tasks
 from validmind.errors import MissingRequiredTestInputError
 
@@ -52,7 +54,9 @@ Prompt:
 
 @tags("llm", "zero_shot", "few_shot")
 @tasks("text_classification", "text_summarization")
-def NegativeInstruction(model, min_threshold=7, judge_llm=None):
+def NegativeInstruction(
+    model, min_threshold=7, judge_llm=None
+) -> Tuple[List[Dict[str, Any]], bool, RawData]:
     """
     Evaluates and grades the use of affirmative, proactive language over negative instructions in LLM prompts.