validmind 2.8.29__py3-none-any.whl → 2.9.2__py3-none-any.whl

validmind/tests/stats/CorrelationAnalysis.py

@@ -0,0 +1,251 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.utils import format_records
+from validmind.vm_models import VMDataset
+
+
+def _validate_and_prepare_data(dataset: VMDataset, columns: Optional[List[str]]):
+    """Validate inputs and prepare data for correlation analysis."""
+    if columns is None:
+        columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        columns = [col for col in columns if col in available_columns]
+
+    if not columns:
+        raise SkipTestError("No numerical columns found for correlation analysis")
+
+    if len(columns) < 2:
+        raise SkipTestError(
+            "At least 2 numerical columns required for correlation analysis"
+        )
+
+    # Get data and remove constant columns
+    data = dataset.df[columns].dropna()
+    data = data.loc[:, data.var() != 0]
+
+    if data.shape[1] < 2:
+        raise SkipTestError(
+            "Insufficient non-constant columns for correlation analysis"
+        )
+
+    return data
+
+
+def _compute_correlation_matrices(data, method: str):
+    """Compute correlation and p-value matrices based on method."""
+    if method == "pearson":
+        return _compute_pearson_with_pvalues(data)
+    elif method == "spearman":
+        return _compute_spearman_with_pvalues(data)
+    elif method == "kendall":
+        return _compute_kendall_with_pvalues(data)
+    else:
+        raise ValueError(f"Unsupported correlation method: {method}")
+
+
+def _create_correlation_pairs(
+    corr_matrix, p_matrix, significance_level: float, min_correlation: float
+):
+    """Create correlation pairs table."""
+    correlation_pairs = []
+
+    for i, col1 in enumerate(corr_matrix.columns):
+        for j, col2 in enumerate(corr_matrix.columns):
+            if i < j:  # Only upper triangle to avoid duplicates
+                corr_val = corr_matrix.iloc[i, j]
+                p_val = p_matrix.iloc[i, j]
+
+                if abs(corr_val) >= min_correlation:
+                    pair_info = {
+                        "Feature 1": col1,
+                        "Feature 2": col2,
+                        "Correlation": corr_val,
+                        "Abs Correlation": abs(corr_val),
+                        "p-value": p_val,
+                        "Significant": "Yes" if p_val < significance_level else "No",
+                        "Strength": _correlation_strength(abs(corr_val)),
+                        "Direction": "Positive" if corr_val > 0 else "Negative",
+                    }
+                    correlation_pairs.append(pair_info)
+
+    # Sort by absolute correlation value
+    correlation_pairs.sort(key=lambda x: x["Abs Correlation"], reverse=True)
+    return correlation_pairs
+
+
+def _create_summary_statistics(corr_matrix, correlation_pairs):
+    """Create summary statistics table."""
+    all_correlations = []
+    for i in range(len(corr_matrix.columns)):
+        for j in range(i + 1, len(corr_matrix.columns)):
+            all_correlations.append(abs(corr_matrix.iloc[i, j]))
+
+    significant_count = sum(
+        1 for pair in correlation_pairs if pair["Significant"] == "Yes"
+    )
+    high_corr_count = sum(
+        1 for pair in correlation_pairs if pair["Abs Correlation"] > 0.7
+    )
+    very_high_corr_count = sum(
+        1 for pair in correlation_pairs if pair["Abs Correlation"] > 0.9
+    )
+
+    return {
+        "Total Feature Pairs": len(all_correlations),
+        "Pairs Above Threshold": len(correlation_pairs),
+        "Significant Correlations": significant_count,
+        "High Correlations (>0.7)": high_corr_count,
+        "Very High Correlations (>0.9)": very_high_corr_count,
+        "Mean Absolute Correlation": np.mean(all_correlations),
+        "Max Absolute Correlation": np.max(all_correlations),
+        "Median Absolute Correlation": np.median(all_correlations),
+    }
+
+
+@tags("tabular_data", "statistics", "correlation")
+@tasks("classification", "regression", "clustering")
+def CorrelationAnalysis(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    method: str = "pearson",
+    significance_level: float = 0.05,
+    min_correlation: float = 0.1,
+) -> Dict[str, Any]:
+    """
+    Performs comprehensive correlation analysis with significance testing for numerical features.
+
+    ### Purpose
+
+    This test conducts detailed correlation analysis between numerical features, including
+    correlation coefficients, significance testing, and identification of significant
+    relationships. It helps identify multicollinearity, feature relationships, and
+    potential redundancies in the dataset.
+
+    ### Test Mechanism
+
+    The test computes correlation coefficients using the specified method and performs
+    statistical significance testing for each correlation pair. It provides:
+    - Correlation matrix with significance indicators
+    - List of significant correlations above threshold
+    - Summary statistics about correlation patterns
+    - Identification of highly correlated feature pairs
+
+    ### Signs of High Risk
+
+    - Very high correlations (>0.9) indicating potential multicollinearity
+    - Many significant correlations suggesting complex feature interactions
+    - Features with no significant correlations to others (potential isolation)
+    - Unexpected correlation patterns contradicting domain knowledge
+
+    ### Strengths
+
+    - Provides statistical significance testing for correlations
+    - Supports multiple correlation methods (Pearson, Spearman, Kendall)
+    - Identifies potentially problematic high correlations
+    - Filters results by minimum correlation threshold
+    - Comprehensive summary of correlation patterns
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Cannot detect non-linear relationships (except with Spearman)
+    - Significance testing assumes certain distributional properties
+    - Correlation does not imply causation
+    """
+    # Validate and prepare data
+    data = _validate_and_prepare_data(dataset, columns)
+
+    # Compute correlation matrices
+    corr_matrix, p_matrix = _compute_correlation_matrices(data, method)
+
+    # Create correlation pairs
+    correlation_pairs = _create_correlation_pairs(
+        corr_matrix, p_matrix, significance_level, min_correlation
+    )
+
+    # Build results
+    results = {}
+    if correlation_pairs:
+        results["Correlation Pairs"] = format_records(pd.DataFrame(correlation_pairs))
+
+    # Create summary statistics
+    summary_stats = _create_summary_statistics(corr_matrix, correlation_pairs)
+    results["Summary Statistics"] = format_records(pd.DataFrame([summary_stats]))
+
+    return results
+
+
+def _compute_pearson_with_pvalues(data):
+    """Compute Pearson correlation with p-values"""
+    n_vars = data.shape[1]
+    corr_matrix = data.corr(method="pearson")
+    p_matrix = pd.DataFrame(
+        np.zeros((n_vars, n_vars)), index=corr_matrix.index, columns=corr_matrix.columns
+    )
+
+    for i, col1 in enumerate(data.columns):
+        for j, col2 in enumerate(data.columns):
+            if i != j:
+                _, p_val = stats.pearsonr(data[col1], data[col2])
+                p_matrix.iloc[i, j] = p_val
+
+    return corr_matrix, p_matrix
+
+
+def _compute_spearman_with_pvalues(data):
+    """Compute Spearman correlation with p-values"""
+    n_vars = data.shape[1]
+    corr_matrix = data.corr(method="spearman")
+    p_matrix = pd.DataFrame(
+        np.zeros((n_vars, n_vars)), index=corr_matrix.index, columns=corr_matrix.columns
+    )
+
+    for i, col1 in enumerate(data.columns):
+        for j, col2 in enumerate(data.columns):
+            if i != j:
+                _, p_val = stats.spearmanr(data[col1], data[col2])
+                p_matrix.iloc[i, j] = p_val
+
+    return corr_matrix, p_matrix
+
+
+def _compute_kendall_with_pvalues(data):
+    """Compute Kendall correlation with p-values"""
+    n_vars = data.shape[1]
+    corr_matrix = data.corr(method="kendall")
+    p_matrix = pd.DataFrame(
+        np.zeros((n_vars, n_vars)), index=corr_matrix.index, columns=corr_matrix.columns
+    )
+
+    for i, col1 in enumerate(data.columns):
+        for j, col2 in enumerate(data.columns):
+            if i != j:
+                _, p_val = stats.kendalltau(data[col1], data[col2])
+                p_matrix.iloc[i, j] = p_val
+
+    return corr_matrix, p_matrix
+
+
+def _correlation_strength(abs_corr):
+    """Classify correlation strength"""
+    if abs_corr >= 0.9:
+        return "Very Strong"
+    elif abs_corr >= 0.7:
+        return "Strong"
+    elif abs_corr >= 0.5:
+        return "Moderate"
+    elif abs_corr >= 0.3:
+        return "Weak"
+    else:
+        return "Very Weak"

validmind/tests/stats/DescriptiveStats.py

@@ -0,0 +1,197 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.utils import format_records
+from validmind.vm_models import VMDataset
+
+
+def _validate_columns(dataset: VMDataset, columns: Optional[List[str]]):
+    """Validate and return numerical columns (excluding boolean columns)."""
+    if columns is None:
+        # Get all columns marked as numeric
+        numeric_columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        numeric_columns = [col for col in columns if col in available_columns]
+
+    # Filter out boolean columns as they can't have proper statistical measures computed
+    columns = []
+    for col in numeric_columns:
+        dtype = dataset.df[col].dtype
+        # Only include integer and float types, exclude boolean
+        if pd.api.types.is_integer_dtype(dtype) or pd.api.types.is_float_dtype(dtype):
+            columns.append(col)
+
+    if not columns:
+        raise SkipTestError(
+            "No numerical columns (integer/float) found for descriptive statistics"
+        )
+
+    return columns
+
+
+def _compute_basic_stats(column: str, data, total_count: int):
+    """Compute basic statistics for a column."""
+    return {
+        "Feature": column,
+        "Count": len(data),
+        "Missing": total_count - len(data),
+        "Missing %": ((total_count - len(data)) / total_count) * 100,
+        "Mean": data.mean(),
+        "Median": data.median(),
+        "Std": data.std(),
+        "Min": data.min(),
+        "Max": data.max(),
+        "Q1": data.quantile(0.25),
+        "Q3": data.quantile(0.75),
+        "IQR": data.quantile(0.75) - data.quantile(0.25),
+    }
+
+
+def _compute_advanced_stats(column: str, data, confidence_level: float):
+    """Compute advanced statistics for a column."""
+    try:
+        # Distribution measures
+        skewness = stats.skew(data)
+        kurtosis_val = stats.kurtosis(data)
+        cv = (data.std() / data.mean()) * 100 if data.mean() != 0 else np.nan
+
+        # Confidence interval for mean
+        ci_lower, ci_upper = stats.t.interval(
+            confidence_level,
+            len(data) - 1,
+            loc=data.mean(),
+            scale=data.std() / np.sqrt(len(data)),
+        )
+
+        # Normality test
+        if len(data) <= 5000:
+            normality_stat, normality_p = stats.shapiro(data)
+            normality_test = "Shapiro-Wilk"
+        else:
+            ad_result = stats.anderson(data, dist="norm")
+            normality_stat = ad_result.statistic
+            normality_p = 0.05 if normality_stat > ad_result.critical_values[2] else 0.1
+            normality_test = "Anderson-Darling"
+
+        # Outlier detection using IQR method
+        iqr = data.quantile(0.75) - data.quantile(0.25)
+        lower_bound = data.quantile(0.25) - 1.5 * iqr
+        upper_bound = data.quantile(0.75) + 1.5 * iqr
+        outliers = data[(data < lower_bound) | (data > upper_bound)]
+        outlier_count = len(outliers)
+        outlier_pct = (outlier_count / len(data)) * 100
+
+        return {
+            "Feature": column,
+            "Skewness": skewness,
+            "Kurtosis": kurtosis_val,
+            "CV %": cv,
+            f"CI Lower ({confidence_level*100:.0f}%)": ci_lower,
+            f"CI Upper ({confidence_level*100:.0f}%)": ci_upper,
+            "Normality Test": normality_test,
+            "Normality Stat": normality_stat,
+            "Normality p-value": normality_p,
+            "Normal Distribution": "Yes" if normality_p > 0.05 else "No",
+            "Outliers (IQR)": outlier_count,
+            "Outliers %": outlier_pct,
+        }
+    except Exception:
+        return None
+
+
+@tags("tabular_data", "statistics", "data_quality")
+@tasks("classification", "regression", "clustering")
+def DescriptiveStats(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    include_advanced: bool = True,
+    confidence_level: float = 0.95,
+) -> Dict[str, Any]:
+    """
+    Provides comprehensive descriptive statistics for numerical features in a dataset.
+
+    ### Purpose
+
+    This test generates detailed descriptive statistics for numerical features, including
+    basic statistics, distribution measures, confidence intervals, and normality tests.
+    It provides a comprehensive overview of data characteristics essential for
+    understanding data quality and distribution properties.
+
+    ### Test Mechanism
+
+    The test computes various statistical measures for each numerical column:
+    - Basic statistics: count, mean, median, std, min, max, quartiles
+    - Distribution measures: skewness, kurtosis, coefficient of variation
+    - Confidence intervals for the mean
+    - Normality tests (Shapiro-Wilk for small samples, Anderson-Darling for larger)
+    - Missing value analysis
+
+    ### Signs of High Risk
+
+    - High skewness or kurtosis indicating non-normal distributions
+    - Large coefficients of variation suggesting high data variability
+    - Significant results in normality tests when normality is expected
+    - High percentage of missing values
+    - Extreme outliers based on IQR analysis
+
+    ### Strengths
+
+    - Comprehensive statistical analysis in a single test
+    - Includes advanced statistical measures beyond basic descriptives
+    - Provides confidence intervals for uncertainty quantification
+    - Handles missing values appropriately
+    - Suitable for both exploratory and confirmatory analysis
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Normality tests may not be meaningful for all data types
+    - Large datasets may make some tests computationally expensive
+    - Interpretation requires statistical knowledge
+    """
+    # Validate inputs
+    columns = _validate_columns(dataset, columns)
+
+    # Compute statistics
+    basic_stats = []
+    advanced_stats = []
+
+    for column in columns:
+        data = dataset.df[column].dropna()
+        total_count = len(dataset.df[column])
+
+        if len(data) == 0:
+            continue
+
+        # Basic statistics
+        basic_row = _compute_basic_stats(column, data, total_count)
+        basic_stats.append(basic_row)
+
+        # Advanced statistics
+        if include_advanced and len(data) > 2:
+            advanced_row = _compute_advanced_stats(column, data, confidence_level)
+            if advanced_row is not None:
+                advanced_stats.append(advanced_row)
+
+    # Format results
+    results = {}
+    if basic_stats:
+        results["Basic Statistics"] = format_records(pd.DataFrame(basic_stats))
+
+    if advanced_stats and include_advanced:
+        results["Advanced Statistics"] = format_records(pd.DataFrame(advanced_stats))
+
+    if not results:
+        raise SkipTestError("Unable to compute statistics for any columns")
+
+    return results

validmind/tests/stats/NormalityTests.py

@@ -0,0 +1,147 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+from scipy import stats
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.utils import format_records
+from validmind.vm_models import VMDataset
+
+
+def _validate_columns(dataset: VMDataset, columns: Optional[List[str]]):
+    """Validate and return numerical columns."""
+    if columns is None:
+        columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        columns = [col for col in columns if col in available_columns]
+
+    if not columns:
+        raise SkipTestError("No numerical columns found for normality testing")
+
+    return columns
+
+
+def _run_shapiro_test(data, tests: List[str], alpha: float):
+    """Run Shapiro-Wilk test if requested and data size is appropriate."""
+    results = {}
+    if "shapiro" in tests and len(data) <= 5000:
+        try:
+            stat, p_value = stats.shapiro(data)
+            results["Shapiro-Wilk Stat"] = stat
+            results["Shapiro-Wilk p-value"] = p_value
+            results["Shapiro-Wilk Normal"] = "Yes" if p_value > alpha else "No"
+        except Exception:
+            results["Shapiro-Wilk Normal"] = "Test Failed"
+    return results
+
+
+def _run_anderson_test(data, tests: List[str]):
+    """Run Anderson-Darling test if requested."""
+    results = {}
+    if "anderson" in tests:
+        try:
+            ad_result = stats.anderson(data, dist="norm")
+            critical_value = ad_result.critical_values[2]  # 5% level
+            results["Anderson-Darling Stat"] = ad_result.statistic
+            results["Anderson-Darling Critical"] = critical_value
+            results["Anderson-Darling Normal"] = (
+                "Yes" if ad_result.statistic < critical_value else "No"
+            )
+        except Exception:
+            results["Anderson-Darling Normal"] = "Test Failed"
+    return results
+
+
+def _run_ks_test(data, tests: List[str], alpha: float):
+    """Run Kolmogorov-Smirnov test if requested."""
+    results = {}
+    if "kstest" in tests:
+        try:
+            standardized = (data - data.mean()) / data.std()
+            stat, p_value = stats.kstest(standardized, "norm")
+            results["KS Test Stat"] = stat
+            results["KS Test p-value"] = p_value
+            results["KS Test Normal"] = "Yes" if p_value > alpha else "No"
+        except Exception:
+            results["KS Test Normal"] = "Test Failed"
+    return results
+
+
+def _process_column_tests(column: str, data, tests: List[str], alpha: float):
+    """Process all normality tests for a single column."""
+    result_row = {"Feature": column, "Sample Size": len(data)}
+
+    # Run individual tests
+    result_row.update(_run_shapiro_test(data, tests, alpha))
+    result_row.update(_run_anderson_test(data, tests))
+    result_row.update(_run_ks_test(data, tests, alpha))
+
+    return result_row
+
+
+@tags("tabular_data", "statistics", "normality")
+@tasks("classification", "regression", "clustering")
+def NormalityTests(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    alpha: float = 0.05,
+    tests: List[str] = ["shapiro", "anderson", "kstest"],
+) -> Dict[str, Any]:
+    """
+    Performs multiple normality tests on numerical features to assess distribution normality.
+
+    ### Purpose
+
+    This test evaluates whether numerical features follow a normal distribution using
+    various statistical tests. Understanding distribution normality is crucial for
+    selecting appropriate statistical methods and model assumptions.
+
+    ### Test Mechanism
+
+    The test applies multiple normality tests:
+    - Shapiro-Wilk test: Best for small to medium samples
+    - Anderson-Darling test: More sensitive to deviations in tails
+    - Kolmogorov-Smirnov test: General goodness-of-fit test
+
+    ### Signs of High Risk
+
+    - Multiple normality tests failing consistently
+    - Very low p-values indicating strong evidence against normality
+    - Conflicting results between different normality tests
+
+    ### Strengths
+
+    - Multiple statistical tests for robust assessment
+    - Clear pass/fail indicators for each test
+    - Suitable for different sample sizes
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Some tests sensitive to sample size
+    - Perfect normality is rare in real data
+    """
+    # Validate inputs
+    columns = _validate_columns(dataset, columns)
+
+    # Process each column
+    normality_results = []
+    for column in columns:
+        data = dataset.df[column].dropna()
+
+        if len(data) >= 3:
+            result_row = _process_column_tests(column, data, tests, alpha)
+            normality_results.append(result_row)
+
+    # Format results
+    results = {}
+    if normality_results:
+        results["Normality Tests"] = format_records(pd.DataFrame(normality_results))
+
+    return results