validmind 2.8.29__py3-none-any.whl → 2.10.0rc1__py3-none-any.whl

validmind/tests/plots/HistogramPlot.py

@@ -0,0 +1,233 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List, Optional, Union
+
+import numpy as np
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+from scipy import stats
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.vm_models import VMDataset
+
+
+def _validate_columns(dataset: VMDataset, columns: Optional[List[str]]):
+    """Validate and return numerical columns."""
+    if columns is None:
+        columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        columns = [col for col in columns if col in available_columns]
+
+    if not columns:
+        raise SkipTestError("No numerical columns found for histogram plotting")
+
+    return columns
+
+
+def _process_column_data(data, log_scale: bool, column: str):
+    """Process column data and return plot data and xlabel."""
+    plot_data = data
+    xlabel = column
+    if log_scale and (data > 0).all():
+        plot_data = np.log10(data)
+        xlabel = f"log10({column})"
+    return plot_data, xlabel
+
+
+def _add_histogram_trace(
+    fig, plot_data, bins, color, opacity, normalize, column, row, col
+):
+    """Add histogram trace to figure."""
+    histnorm = "probability density" if normalize else None
+
+    fig.add_trace(
+        go.Histogram(
+            x=plot_data,
+            nbinsx=bins if isinstance(bins, int) else None,
+            name=f"Histogram - {column}",
+            marker_color=color,
+            opacity=opacity,
+            histnorm=histnorm,
+            showlegend=False,
+        ),
+        row=row,
+        col=col,
+    )
+
+
+def _add_kde_trace(fig, plot_data, bins, normalize, column, row, col):
+    """Add KDE trace to figure if possible."""
+    try:
+        kde = stats.gaussian_kde(plot_data)
+        x_range = np.linspace(plot_data.min(), plot_data.max(), 100)
+        kde_values = kde(x_range)
+
+        if not normalize:
+            hist_max = (
+                len(plot_data) / bins if isinstance(bins, int) else len(plot_data) / 30
+            )
+            kde_values = kde_values * hist_max / kde_values.max()
+
+        fig.add_trace(
+            go.Scatter(
+                x=x_range,
+                y=kde_values,
+                mode="lines",
+                name=f"KDE - {column}",
+                line=dict(color="red", width=2),
+                showlegend=False,
+            ),
+            row=row,
+            col=col,
+        )
+    except Exception:
+        pass
+
+
+def _add_stats_annotation(fig, data, idx, row, col):
+    """Add statistics annotation to subplot."""
+    stats_text = f"Mean: {data.mean():.3f}<br>Std: {data.std():.3f}<br>N: {len(data)}"
+    fig.add_annotation(
+        text=stats_text,
+        x=0.02,
+        y=0.98,
+        xref=f"x{idx+1} domain" if idx > 0 else "x domain",
+        yref=f"y{idx+1} domain" if idx > 0 else "y domain",
+        showarrow=False,
+        align="left",
+        bgcolor="rgba(255,255,255,0.8)",
+        bordercolor="black",
+        borderwidth=1,
+        row=row,
+        col=col,
+    )
+
+
+@tags("tabular_data", "visualization", "data_quality")
+@tasks("classification", "regression", "clustering")
+def HistogramPlot(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    bins: Union[int, str, List] = 30,
+    color: str = "steelblue",
+    opacity: float = 0.7,
+    show_kde: bool = True,
+    normalize: bool = False,
+    log_scale: bool = False,
+    title_prefix: str = "Histogram of",
+    width: int = 1200,
+    height: int = 800,
+    n_cols: int = 2,
+    vertical_spacing: float = 0.15,
+    horizontal_spacing: float = 0.1,
+) -> go.Figure:
+    """
+    Generates customizable histogram plots for numerical features in a dataset using Plotly.
+
+    ### Purpose
+
+    This test provides a flexible way to visualize the distribution of numerical features in a dataset.
+    It allows for extensive customization of the histogram appearance and behavior through parameters,
+    making it suitable for various exploratory data analysis tasks.
+
+    ### Test Mechanism
+
+    The test creates histogram plots for specified numerical columns (or all numerical columns if none specified).
+    It supports various customization options including:
+    - Number of bins or bin edges
+    - Color and opacity
+    - Kernel density estimation overlay
+    - Logarithmic scaling
+    - Normalization options
+    - Configurable subplot layout (columns and spacing)
+
+    ### Signs of High Risk
+
+    - Highly skewed distributions that may indicate data quality issues
+    - Unexpected bimodal or multimodal distributions
+    - Presence of extreme outliers
+    - Empty or sparse distributions
+
+    ### Strengths
+
+    - Highly customizable visualization options
+    - Interactive Plotly plots with zoom, pan, and hover capabilities
+    - Supports both single and multiple column analysis
+    - Provides insights into data distribution patterns
+    - Can handle different data types and scales
+    - Configurable subplot layout for better visualization
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Visual interpretation may be subjective
+    - May not be suitable for high-dimensional datasets
+    - Performance may degrade with very large datasets
+    """
+    # Validate inputs
+    columns = _validate_columns(dataset, columns)
+
+    # Calculate subplot layout
+    n_cols = min(n_cols, len(columns))
+    n_rows = (len(columns) + n_cols - 1) // n_cols
+
+    # Create subplots
+    subplot_titles = [f"{title_prefix} {col}" for col in columns]
+    fig = make_subplots(
+        rows=n_rows,
+        cols=n_cols,
+        subplot_titles=subplot_titles,
+        vertical_spacing=vertical_spacing,
+        horizontal_spacing=horizontal_spacing,
+    )
+
+    for idx, column in enumerate(columns):
+        row = (idx // n_cols) + 1
+        col = (idx % n_cols) + 1
+        data = dataset.df[column].dropna()
+
+        if len(data) == 0:
+            fig.add_annotation(
+                text=f"No data available<br>for {column}",
+                x=0.5,
+                y=0.5,
+                xref=f"x{idx+1}" if idx > 0 else "x",
+                yref=f"y{idx+1}" if idx > 0 else "y",
+                showarrow=False,
+                row=row,
+                col=col,
+            )
+            continue
+
+        # Process data
+        plot_data, xlabel = _process_column_data(data, log_scale, column)
+
+        # Add histogram
+        _add_histogram_trace(
+            fig, plot_data, bins, color, opacity, normalize, column, row, col
+        )
+
+        # Add KDE if requested
+        if show_kde and len(data) > 1:
+            _add_kde_trace(fig, plot_data, bins, normalize, column, row, col)
+
+        # Update axes and add annotations
+        fig.update_xaxes(title_text=xlabel, row=row, col=col)
+        ylabel = "Density" if normalize else "Frequency"
+        fig.update_yaxes(title_text=ylabel, row=row, col=col)
+        _add_stats_annotation(fig, data, idx, row, col)
+
+    # Update layout
+    fig.update_layout(
+        title_text="Dataset Feature Distributions",
+        showlegend=False,
+        width=width,
+        height=height,
+        template="plotly_white",
+    )
+
+    return fig

validmind/tests/plots/ViolinPlot.py

@@ -0,0 +1,125 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List, Optional
+
+import plotly.express as px
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.vm_models import VMDataset
+
+
+@tags("tabular_data", "visualization", "distribution")
+@tasks("classification", "regression", "clustering")
+def ViolinPlot(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    group_by: Optional[str] = None,
+    width: int = 800,
+    height: int = 600,
+) -> px.violin:
+    """
+    Generates interactive violin plots for numerical features using Plotly.
+
+    ### Purpose
+
+    This test creates violin plots to visualize the distribution of numerical features,
+    showing both the probability density and summary statistics. Violin plots combine
+    aspects of box plots and kernel density estimation for rich distribution visualization.
+
+    ### Test Mechanism
+
+    The test creates violin plots for specified numerical columns, with optional
+    grouping by categorical variables. Each violin shows the distribution shape,
+    quartiles, and median values.
+
+    ### Signs of High Risk
+
+    - Multimodal distributions that might indicate mixed populations
+    - Highly skewed distributions suggesting data quality issues
+    - Large differences in distribution shapes across groups
+    - Unusual distribution patterns that contradict domain expectations
+
+    ### Strengths
+
+    - Shows detailed distribution shape information
+    - Interactive Plotly visualization with hover details
+    - Effective for comparing distributions across groups
+    - Combines density estimation with quartile information
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Requires sufficient data points for meaningful density estimation
+    - May not be suitable for discrete variables
+    - Can be misleading with very small sample sizes
+    """
+    # Get numerical columns
+    if columns is None:
+        columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        columns = [col for col in columns if col in available_columns]
+
+    if not columns:
+        raise SkipTestError("No numerical columns found for violin plot")
+
+    # For violin plots, we'll melt the data to long format
+    data = dataset.df[columns].dropna()
+
+    if len(data) == 0:
+        raise SkipTestError("No valid data available for violin plot")
+
+    # Melt the dataframe to long format
+    melted_data = data.melt(var_name="Feature", value_name="Value")
+
+    # Add group column if specified
+    if group_by and group_by in dataset.df.columns:
+        # Repeat group values for each feature
+        group_values = []
+        for column in columns:
+            column_data = dataset.df[[column, group_by]].dropna()
+            group_values.extend(column_data[group_by].tolist())
+
+        if len(group_values) == len(melted_data):
+            melted_data["Group"] = group_values
+        else:
+            group_by = None  # Disable grouping if lengths don't match
+
+    # Create violin plot
+    if group_by and "Group" in melted_data.columns:
+        fig = px.violin(
+            melted_data,
+            x="Feature",
+            y="Value",
+            color="Group",
+            box=True,
+            title=f"Distribution of Features by {group_by}",
+            width=width,
+            height=height,
+        )
+    else:
+        fig = px.violin(
+            melted_data,
+            x="Feature",
+            y="Value",
+            box=True,
+            title="Feature Distributions",
+            width=width,
+            height=height,
+        )
+
+    # Update layout
+    fig.update_layout(
+        template="plotly_white",
+        title_x=0.5,
+        xaxis_title="Features",
+        yaxis_title="Values",
+    )
+
+    # Rotate x-axis labels for better readability
+    fig.update_xaxes(tickangle=45)
+
+    return fig

validmind/tests/stats/CorrelationAnalysis.py

@@ -0,0 +1,251 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.utils import format_records
+from validmind.vm_models import VMDataset
+
+
+def _validate_and_prepare_data(dataset: VMDataset, columns: Optional[List[str]]):
+    """Validate inputs and prepare data for correlation analysis."""
+    if columns is None:
+        columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        columns = [col for col in columns if col in available_columns]
+
+    if not columns:
+        raise SkipTestError("No numerical columns found for correlation analysis")
+
+    if len(columns) < 2:
+        raise SkipTestError(
+            "At least 2 numerical columns required for correlation analysis"
+        )
+
+    # Get data and remove constant columns
+    data = dataset.df[columns].dropna()
+    data = data.loc[:, data.var() != 0]
+
+    if data.shape[1] < 2:
+        raise SkipTestError(
+            "Insufficient non-constant columns for correlation analysis"
+        )
+
+    return data
+
+
+def _compute_correlation_matrices(data, method: str):
+    """Compute correlation and p-value matrices based on method."""
+    if method == "pearson":
+        return _compute_pearson_with_pvalues(data)
+    elif method == "spearman":
+        return _compute_spearman_with_pvalues(data)
+    elif method == "kendall":
+        return _compute_kendall_with_pvalues(data)
+    else:
+        raise ValueError(f"Unsupported correlation method: {method}")
+
+
+def _create_correlation_pairs(
+    corr_matrix, p_matrix, significance_level: float, min_correlation: float
+):
+    """Create correlation pairs table."""
+    correlation_pairs = []
+
+    for i, col1 in enumerate(corr_matrix.columns):
+        for j, col2 in enumerate(corr_matrix.columns):
+            if i < j:  # Only upper triangle to avoid duplicates
+                corr_val = corr_matrix.iloc[i, j]
+                p_val = p_matrix.iloc[i, j]
+
+                if abs(corr_val) >= min_correlation:
+                    pair_info = {
+                        "Feature 1": col1,
+                        "Feature 2": col2,
+                        "Correlation": corr_val,
+                        "Abs Correlation": abs(corr_val),
+                        "p-value": p_val,
+                        "Significant": "Yes" if p_val < significance_level else "No",
+                        "Strength": _correlation_strength(abs(corr_val)),
+                        "Direction": "Positive" if corr_val > 0 else "Negative",
+                    }
+                    correlation_pairs.append(pair_info)
+
+    # Sort by absolute correlation value
+    correlation_pairs.sort(key=lambda x: x["Abs Correlation"], reverse=True)
+    return correlation_pairs
+
+
+def _create_summary_statistics(corr_matrix, correlation_pairs):
+    """Create summary statistics table."""
+    all_correlations = []
+    for i in range(len(corr_matrix.columns)):
+        for j in range(i + 1, len(corr_matrix.columns)):
+            all_correlations.append(abs(corr_matrix.iloc[i, j]))
+
+    significant_count = sum(
+        1 for pair in correlation_pairs if pair["Significant"] == "Yes"
+    )
+    high_corr_count = sum(
+        1 for pair in correlation_pairs if pair["Abs Correlation"] > 0.7
+    )
+    very_high_corr_count = sum(
+        1 for pair in correlation_pairs if pair["Abs Correlation"] > 0.9
+    )
+
+    return {
+        "Total Feature Pairs": len(all_correlations),
+        "Pairs Above Threshold": len(correlation_pairs),
+        "Significant Correlations": significant_count,
+        "High Correlations (>0.7)": high_corr_count,
+        "Very High Correlations (>0.9)": very_high_corr_count,
+        "Mean Absolute Correlation": np.mean(all_correlations),
+        "Max Absolute Correlation": np.max(all_correlations),
+        "Median Absolute Correlation": np.median(all_correlations),
+    }
+
+
+@tags("tabular_data", "statistics", "correlation")
+@tasks("classification", "regression", "clustering")
+def CorrelationAnalysis(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    method: str = "pearson",
+    significance_level: float = 0.05,
+    min_correlation: float = 0.1,
+) -> Dict[str, Any]:
+    """
+    Performs comprehensive correlation analysis with significance testing for numerical features.
+
+    ### Purpose
+
+    This test conducts detailed correlation analysis between numerical features, including
+    correlation coefficients, significance testing, and identification of significant
+    relationships. It helps identify multicollinearity, feature relationships, and
+    potential redundancies in the dataset.
+
+    ### Test Mechanism
+
+    The test computes correlation coefficients using the specified method and performs
+    statistical significance testing for each correlation pair. It provides:
+    - Correlation matrix with significance indicators
+    - List of significant correlations above threshold
+    - Summary statistics about correlation patterns
+    - Identification of highly correlated feature pairs
+
+    ### Signs of High Risk
+
+    - Very high correlations (>0.9) indicating potential multicollinearity
+    - Many significant correlations suggesting complex feature interactions
+    - Features with no significant correlations to others (potential isolation)
+    - Unexpected correlation patterns contradicting domain knowledge
+
+    ### Strengths
+
+    - Provides statistical significance testing for correlations
+    - Supports multiple correlation methods (Pearson, Spearman, Kendall)
+    - Identifies potentially problematic high correlations
+    - Filters results by minimum correlation threshold
+    - Comprehensive summary of correlation patterns
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Cannot detect non-linear relationships (except with Spearman)
+    - Significance testing assumes certain distributional properties
+    - Correlation does not imply causation
+    """
+    # Validate and prepare data
+    data = _validate_and_prepare_data(dataset, columns)
+
+    # Compute correlation matrices
+    corr_matrix, p_matrix = _compute_correlation_matrices(data, method)
+
+    # Create correlation pairs
+    correlation_pairs = _create_correlation_pairs(
+        corr_matrix, p_matrix, significance_level, min_correlation
+    )
+
+    # Build results
+    results = {}
+    if correlation_pairs:
+        results["Correlation Pairs"] = format_records(pd.DataFrame(correlation_pairs))
+
+    # Create summary statistics
+    summary_stats = _create_summary_statistics(corr_matrix, correlation_pairs)
+    results["Summary Statistics"] = format_records(pd.DataFrame([summary_stats]))
+
+    return results
+
+
+def _compute_pearson_with_pvalues(data):
+    """Compute Pearson correlation with p-values"""
+    n_vars = data.shape[1]
+    corr_matrix = data.corr(method="pearson")
+    p_matrix = pd.DataFrame(
+        np.zeros((n_vars, n_vars)), index=corr_matrix.index, columns=corr_matrix.columns
+    )
+
+    for i, col1 in enumerate(data.columns):
+        for j, col2 in enumerate(data.columns):
+            if i != j:
+                _, p_val = stats.pearsonr(data[col1], data[col2])
+                p_matrix.iloc[i, j] = p_val
+
+    return corr_matrix, p_matrix
+
+
+def _compute_spearman_with_pvalues(data):
+    """Compute Spearman correlation with p-values"""
+    n_vars = data.shape[1]
+    corr_matrix = data.corr(method="spearman")
+    p_matrix = pd.DataFrame(
+        np.zeros((n_vars, n_vars)), index=corr_matrix.index, columns=corr_matrix.columns
+    )
+
+    for i, col1 in enumerate(data.columns):
+        for j, col2 in enumerate(data.columns):
+            if i != j:
+                _, p_val = stats.spearmanr(data[col1], data[col2])
+                p_matrix.iloc[i, j] = p_val
+
+    return corr_matrix, p_matrix
+
+
+def _compute_kendall_with_pvalues(data):
+    """Compute Kendall correlation with p-values"""
+    n_vars = data.shape[1]
+    corr_matrix = data.corr(method="kendall")
+    p_matrix = pd.DataFrame(
+        np.zeros((n_vars, n_vars)), index=corr_matrix.index, columns=corr_matrix.columns
+    )
+
+    for i, col1 in enumerate(data.columns):
+        for j, col2 in enumerate(data.columns):
+            if i != j:
+                _, p_val = stats.kendalltau(data[col1], data[col2])
+                p_matrix.iloc[i, j] = p_val
+
+    return corr_matrix, p_matrix
+
+
+def _correlation_strength(abs_corr):
+    """Classify correlation strength"""
+    if abs_corr >= 0.9:
+        return "Very Strong"
+    elif abs_corr >= 0.7:
+        return "Strong"
+    elif abs_corr >= 0.5:
+        return "Moderate"
+    elif abs_corr >= 0.3:
+        return "Weak"
+    else:
+        return "Very Weak"