validmind 2.8.29__py3-none-any.whl → 2.10.0rc1__py3-none-any.whl

validmind/tests/stats/DescriptiveStats.py

@@ -0,0 +1,197 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.utils import format_records
+from validmind.vm_models import VMDataset
+
+
+def _validate_columns(dataset: VMDataset, columns: Optional[List[str]]):
+    """Validate and return numerical columns (excluding boolean columns)."""
+    if columns is None:
+        # Get all columns marked as numeric
+        numeric_columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        numeric_columns = [col for col in columns if col in available_columns]
+
+    # Filter out boolean columns as they can't have proper statistical measures computed
+    columns = []
+    for col in numeric_columns:
+        dtype = dataset.df[col].dtype
+        # Only include integer and float types, exclude boolean
+        if pd.api.types.is_integer_dtype(dtype) or pd.api.types.is_float_dtype(dtype):
+            columns.append(col)
+
+    if not columns:
+        raise SkipTestError(
+            "No numerical columns (integer/float) found for descriptive statistics"
+        )
+
+    return columns
+
+
+def _compute_basic_stats(column: str, data, total_count: int):
+    """Compute basic statistics for a column."""
+    return {
+        "Feature": column,
+        "Count": len(data),
+        "Missing": total_count - len(data),
+        "Missing %": ((total_count - len(data)) / total_count) * 100,
+        "Mean": data.mean(),
+        "Median": data.median(),
+        "Std": data.std(),
+        "Min": data.min(),
+        "Max": data.max(),
+        "Q1": data.quantile(0.25),
+        "Q3": data.quantile(0.75),
+        "IQR": data.quantile(0.75) - data.quantile(0.25),
+    }
+
+
+def _compute_advanced_stats(column: str, data, confidence_level: float):
+    """Compute advanced statistics for a column."""
+    try:
+        # Distribution measures
+        skewness = stats.skew(data)
+        kurtosis_val = stats.kurtosis(data)
+        cv = (data.std() / data.mean()) * 100 if data.mean() != 0 else np.nan
+
+        # Confidence interval for mean
+        ci_lower, ci_upper = stats.t.interval(
+            confidence_level,
+            len(data) - 1,
+            loc=data.mean(),
+            scale=data.std() / np.sqrt(len(data)),
+        )
+
+        # Normality test
+        if len(data) <= 5000:
+            normality_stat, normality_p = stats.shapiro(data)
+            normality_test = "Shapiro-Wilk"
+        else:
+            ad_result = stats.anderson(data, dist="norm")
+            normality_stat = ad_result.statistic
+            normality_p = 0.05 if normality_stat > ad_result.critical_values[2] else 0.1
+            normality_test = "Anderson-Darling"
+
+        # Outlier detection using IQR method
+        iqr = data.quantile(0.75) - data.quantile(0.25)
+        lower_bound = data.quantile(0.25) - 1.5 * iqr
+        upper_bound = data.quantile(0.75) + 1.5 * iqr
+        outliers = data[(data < lower_bound) | (data > upper_bound)]
+        outlier_count = len(outliers)
+        outlier_pct = (outlier_count / len(data)) * 100
+
+        return {
+            "Feature": column,
+            "Skewness": skewness,
+            "Kurtosis": kurtosis_val,
+            "CV %": cv,
+            f"CI Lower ({confidence_level*100:.0f}%)": ci_lower,
+            f"CI Upper ({confidence_level*100:.0f}%)": ci_upper,
+            "Normality Test": normality_test,
+            "Normality Stat": normality_stat,
+            "Normality p-value": normality_p,
+            "Normal Distribution": "Yes" if normality_p > 0.05 else "No",
+            "Outliers (IQR)": outlier_count,
+            "Outliers %": outlier_pct,
+        }
+    except Exception:
+        return None
+
+
+@tags("tabular_data", "statistics", "data_quality")
+@tasks("classification", "regression", "clustering")
+def DescriptiveStats(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    include_advanced: bool = True,
+    confidence_level: float = 0.95,
+) -> Dict[str, Any]:
+    """
+    Provides comprehensive descriptive statistics for numerical features in a dataset.
+
+    ### Purpose
+
+    This test generates detailed descriptive statistics for numerical features, including
+    basic statistics, distribution measures, confidence intervals, and normality tests.
+    It provides a comprehensive overview of data characteristics essential for
+    understanding data quality and distribution properties.
+
+    ### Test Mechanism
+
+    The test computes various statistical measures for each numerical column:
+    - Basic statistics: count, mean, median, std, min, max, quartiles
+    - Distribution measures: skewness, kurtosis, coefficient of variation
+    - Confidence intervals for the mean
+    - Normality tests (Shapiro-Wilk for small samples, Anderson-Darling for larger)
+    - Missing value analysis
+
+    ### Signs of High Risk
+
+    - High skewness or kurtosis indicating non-normal distributions
+    - Large coefficients of variation suggesting high data variability
+    - Significant results in normality tests when normality is expected
+    - High percentage of missing values
+    - Extreme outliers based on IQR analysis
+
+    ### Strengths
+
+    - Comprehensive statistical analysis in a single test
+    - Includes advanced statistical measures beyond basic descriptives
+    - Provides confidence intervals for uncertainty quantification
+    - Handles missing values appropriately
+    - Suitable for both exploratory and confirmatory analysis
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Normality tests may not be meaningful for all data types
+    - Large datasets may make some tests computationally expensive
+    - Interpretation requires statistical knowledge
+    """
+    # Validate inputs
+    columns = _validate_columns(dataset, columns)
+
+    # Compute statistics
+    basic_stats = []
+    advanced_stats = []
+
+    for column in columns:
+        data = dataset.df[column].dropna()
+        total_count = len(dataset.df[column])
+
+        if len(data) == 0:
+            continue
+
+        # Basic statistics
+        basic_row = _compute_basic_stats(column, data, total_count)
+        basic_stats.append(basic_row)
+
+        # Advanced statistics
+        if include_advanced and len(data) > 2:
+            advanced_row = _compute_advanced_stats(column, data, confidence_level)
+            if advanced_row is not None:
+                advanced_stats.append(advanced_row)
+
+    # Format results
+    results = {}
+    if basic_stats:
+        results["Basic Statistics"] = format_records(pd.DataFrame(basic_stats))
+
+    if advanced_stats and include_advanced:
+        results["Advanced Statistics"] = format_records(pd.DataFrame(advanced_stats))
+
+    if not results:
+        raise SkipTestError("Unable to compute statistics for any columns")
+
+    return results

validmind/tests/stats/NormalityTests.py

@@ -0,0 +1,147 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+from scipy import stats
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.utils import format_records
+from validmind.vm_models import VMDataset
+
+
+def _validate_columns(dataset: VMDataset, columns: Optional[List[str]]):
+    """Validate and return numerical columns."""
+    if columns is None:
+        columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        columns = [col for col in columns if col in available_columns]
+
+    if not columns:
+        raise SkipTestError("No numerical columns found for normality testing")
+
+    return columns
+
+
+def _run_shapiro_test(data, tests: List[str], alpha: float):
+    """Run Shapiro-Wilk test if requested and data size is appropriate."""
+    results = {}
+    if "shapiro" in tests and len(data) <= 5000:
+        try:
+            stat, p_value = stats.shapiro(data)
+            results["Shapiro-Wilk Stat"] = stat
+            results["Shapiro-Wilk p-value"] = p_value
+            results["Shapiro-Wilk Normal"] = "Yes" if p_value > alpha else "No"
+        except Exception:
+            results["Shapiro-Wilk Normal"] = "Test Failed"
+    return results
+
+
+def _run_anderson_test(data, tests: List[str]):
+    """Run Anderson-Darling test if requested."""
+    results = {}
+    if "anderson" in tests:
+        try:
+            ad_result = stats.anderson(data, dist="norm")
+            critical_value = ad_result.critical_values[2]  # 5% level
+            results["Anderson-Darling Stat"] = ad_result.statistic
+            results["Anderson-Darling Critical"] = critical_value
+            results["Anderson-Darling Normal"] = (
+                "Yes" if ad_result.statistic < critical_value else "No"
+            )
+        except Exception:
+            results["Anderson-Darling Normal"] = "Test Failed"
+    return results
+
+
+def _run_ks_test(data, tests: List[str], alpha: float):
+    """Run Kolmogorov-Smirnov test if requested."""
+    results = {}
+    if "kstest" in tests:
+        try:
+            standardized = (data - data.mean()) / data.std()
+            stat, p_value = stats.kstest(standardized, "norm")
+            results["KS Test Stat"] = stat
+            results["KS Test p-value"] = p_value
+            results["KS Test Normal"] = "Yes" if p_value > alpha else "No"
+        except Exception:
+            results["KS Test Normal"] = "Test Failed"
+    return results
+
+
+def _process_column_tests(column: str, data, tests: List[str], alpha: float):
+    """Process all normality tests for a single column."""
+    result_row = {"Feature": column, "Sample Size": len(data)}
+
+    # Run individual tests
+    result_row.update(_run_shapiro_test(data, tests, alpha))
+    result_row.update(_run_anderson_test(data, tests))
+    result_row.update(_run_ks_test(data, tests, alpha))
+
+    return result_row
+
+
+@tags("tabular_data", "statistics", "normality")
+@tasks("classification", "regression", "clustering")
+def NormalityTests(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    alpha: float = 0.05,
+    tests: List[str] = ["shapiro", "anderson", "kstest"],
+) -> Dict[str, Any]:
+    """
+    Performs multiple normality tests on numerical features to assess distribution normality.
+
+    ### Purpose
+
+    This test evaluates whether numerical features follow a normal distribution using
+    various statistical tests. Understanding distribution normality is crucial for
+    selecting appropriate statistical methods and model assumptions.
+
+    ### Test Mechanism
+
+    The test applies multiple normality tests:
+    - Shapiro-Wilk test: Best for small to medium samples
+    - Anderson-Darling test: More sensitive to deviations in tails
+    - Kolmogorov-Smirnov test: General goodness-of-fit test
+
+    ### Signs of High Risk
+
+    - Multiple normality tests failing consistently
+    - Very low p-values indicating strong evidence against normality
+    - Conflicting results between different normality tests
+
+    ### Strengths
+
+    - Multiple statistical tests for robust assessment
+    - Clear pass/fail indicators for each test
+    - Suitable for different sample sizes
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Some tests sensitive to sample size
+    - Perfect normality is rare in real data
+    """
+    # Validate inputs
+    columns = _validate_columns(dataset, columns)
+
+    # Process each column
+    normality_results = []
+    for column in columns:
+        data = dataset.df[column].dropna()
+
+        if len(data) >= 3:
+            result_row = _process_column_tests(column, data, tests, alpha)
+            normality_results.append(result_row)
+
+    # Format results
+    results = {}
+    if normality_results:
+        results["Normality Tests"] = format_records(pd.DataFrame(normality_results))
+
+    return results

validmind/tests/stats/OutlierDetection.py

@@ -0,0 +1,173 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+from sklearn.ensemble import IsolationForest
+
+from validmind import tags, tasks
+from validmind.errors import SkipTestError
+from validmind.utils import format_records
+from validmind.vm_models import VMDataset
+
+
+def _validate_columns(dataset: VMDataset, columns: Optional[List[str]]):
+    """Validate and return numerical columns."""
+    if columns is None:
+        columns = dataset.feature_columns_numeric
+    else:
+        available_columns = set(dataset.feature_columns_numeric)
+        columns = [col for col in columns if col in available_columns]
+
+    # Filter out boolean columns as they can't be used for outlier detection
+    numeric_columns = []
+    for col in columns:
+        if col in dataset.df.columns:
+            col_dtype = dataset.df[col].dtype
+            # Exclude boolean and object types, keep only true numeric types
+            if pd.api.types.is_numeric_dtype(col_dtype) and col_dtype != bool:
+                numeric_columns.append(col)
+
+    columns = numeric_columns
+
+    if not columns:
+        raise SkipTestError("No suitable numerical columns found for outlier detection")
+
+    return columns
+
+
+def _detect_iqr_outliers(data, iqr_threshold: float):
+    """Detect outliers using IQR method."""
+    q1, q3 = data.quantile(0.25), data.quantile(0.75)
+    iqr = q3 - q1
+    lower_bound = q1 - iqr_threshold * iqr
+    upper_bound = q3 + iqr_threshold * iqr
+    # Fix numpy boolean operation error by using pandas boolean indexing properly
+    outlier_mask = (data < lower_bound) | (data > upper_bound)
+    iqr_outliers = data[outlier_mask]
+    return len(iqr_outliers), (len(iqr_outliers) / len(data)) * 100
+
+
+def _detect_zscore_outliers(data, zscore_threshold: float):
+    """Detect outliers using Z-score method."""
+    z_scores = np.abs(stats.zscore(data))
+    # Fix potential numpy boolean operation error
+    outlier_mask = z_scores > zscore_threshold
+    zscore_outliers = data[outlier_mask]
+    return len(zscore_outliers), (len(zscore_outliers) / len(data)) * 100
+
+
+def _detect_isolation_forest_outliers(data, contamination: float):
+    """Detect outliers using Isolation Forest method."""
+    if len(data) <= 10:
+        return 0, 0
+
+    try:
+        iso_forest = IsolationForest(contamination=contamination, random_state=42)
+        outlier_pred = iso_forest.fit_predict(data.values.reshape(-1, 1))
+        iso_outliers = data[outlier_pred == -1]
+        return len(iso_outliers), (len(iso_outliers) / len(data)) * 100
+    except Exception:
+        return 0, 0
+
+
+def _process_column_outliers(
+    column: str,
+    data,
+    methods: List[str],
+    iqr_threshold: float,
+    zscore_threshold: float,
+    contamination: float,
+):
+    """Process outlier detection for a single column."""
+    outliers_dict = {"Feature": column, "Total Count": len(data)}
+
+    # IQR method
+    if "iqr" in methods:
+        count, percentage = _detect_iqr_outliers(data, iqr_threshold)
+        outliers_dict["IQR Outliers"] = count
+        outliers_dict["IQR %"] = percentage
+
+    # Z-score method
+    if "zscore" in methods:
+        count, percentage = _detect_zscore_outliers(data, zscore_threshold)
+        outliers_dict["Z-Score Outliers"] = count
+        outliers_dict["Z-Score %"] = percentage
+
+    # Isolation Forest method
+    if "isolation_forest" in methods:
+        count, percentage = _detect_isolation_forest_outliers(data, contamination)
+        outliers_dict["Isolation Forest Outliers"] = count
+        outliers_dict["Isolation Forest %"] = percentage
+
+    return outliers_dict
+
+
+@tags("tabular_data", "statistics", "outliers")
+@tasks("classification", "regression", "clustering")
+def OutlierDetection(
+    dataset: VMDataset,
+    columns: Optional[List[str]] = None,
+    methods: List[str] = ["iqr", "zscore", "isolation_forest"],
+    iqr_threshold: float = 1.5,
+    zscore_threshold: float = 3.0,
+    contamination: float = 0.1,
+) -> Dict[str, Any]:
+    """
+    Detects outliers in numerical features using multiple statistical methods.
+
+    ### Purpose
+
+    This test identifies outliers in numerical features using various statistical
+    methods including IQR, Z-score, and Isolation Forest. It provides comprehensive
+    outlier detection to help identify data quality issues and potential anomalies.
+
+    ### Test Mechanism
+
+    The test applies multiple outlier detection methods:
+    - IQR method: Values beyond Q1 - 1.5*IQR or Q3 + 1.5*IQR
+    - Z-score method: Values with |z-score| > threshold
+    - Isolation Forest: ML-based anomaly detection
+
+    ### Signs of High Risk
+
+    - High percentage of outliers indicating data quality issues
+    - Inconsistent outlier detection across methods
+    - Extreme outliers that significantly deviate from normal patterns
+
+    ### Strengths
+
+    - Multiple detection methods for robust outlier identification
+    - Customizable thresholds for different sensitivity levels
+    - Clear summary of outlier patterns across features
+
+    ### Limitations
+
+    - Limited to numerical features only
+    - Some methods assume normal distributions
+    - Threshold selection can be subjective
+    """
+    # Validate inputs
+    columns = _validate_columns(dataset, columns)
+
+    # Process each column
+    outlier_summary = []
+    for column in columns:
+        data = dataset._df[column].dropna()
+
+        if len(data) >= 3:
+            outliers_dict = _process_column_outliers(
+                column, data, methods, iqr_threshold, zscore_threshold, contamination
+            )
+            outlier_summary.append(outliers_dict)
+
+    # Format results
+    results = {}
+    if outlier_summary:
+        results["Outlier Summary"] = format_records(pd.DataFrame(outlier_summary))
+
+    return results

validmind/unit_metrics/classification/individual/AbsoluteError.py

@@ -0,0 +1,42 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def AbsoluteError(model: VMModel, dataset: VMDataset, **kwargs) -> List[float]:
+    """Calculates the absolute error per row for a classification model.
+
+    For classification tasks, this computes the absolute difference between
+    the true class labels and predicted class labels for each individual row.
+    For binary classification with probabilities, it can also compute the
+    absolute difference between true labels and predicted probabilities.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predictions
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row absolute errors as a list of float values
+    """
+    y_true = dataset.y
+    y_pred = dataset.y_pred(model)
+
+    # Convert to numpy arrays and ensure same data type
+    y_true = np.asarray(y_true)
+    y_pred = np.asarray(y_pred)
+
+    # For classification, compute absolute difference between true and predicted labels
+    absolute_errors = np.abs(y_true - y_pred)
+
+    # Return as a list of floats
+    return absolute_errors.astype(float).tolist()

validmind/unit_metrics/classification/individual/BrierScore.py

@@ -0,0 +1,56 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def BrierScore(model: VMModel, dataset: VMDataset, **kwargs) -> List[float]:
+    """Calculates the Brier score per row for a classification model.
+
+    The Brier score is a proper score function that measures the accuracy of
+    probabilistic predictions. It is calculated as the mean squared difference
+    between predicted probabilities and the actual binary outcomes.
+    Lower scores indicate better calibration.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predicted probabilities
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row Brier scores as a list of float values
+
+    Raises:
+        ValueError: If probability column is not found for the model
+    """
+    y_true = dataset.y
+
+    # Try to get probabilities
+    try:
+        y_prob = dataset.y_prob(model)
+        # For binary classification, use the positive class probability
+        if y_prob.ndim > 1 and y_prob.shape[1] > 1:
+            y_prob = y_prob[:, 1]  # Use probability of positive class
+    except ValueError:
+        # Fall back to predictions if probabilities not available
+        # Convert predictions to "probabilities" (1.0 for predicted class, 0.0 for other)
+        y_pred = dataset.y_pred(model)
+        y_prob = y_pred.astype(float)
+
+    # Convert to numpy arrays and ensure same data type
+    y_true = np.asarray(y_true, dtype=float)
+    y_prob = np.asarray(y_prob, dtype=float)
+
+    # Calculate Brier score per row: (predicted_probability - actual_outcome)²
+    brier_scores = (y_prob - y_true) ** 2
+
+    # Return as a list of floats
+    return brier_scores.tolist()

validmind/unit_metrics/classification/individual/CalibrationError.py

@@ -0,0 +1,77 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+from typing import List
+
+import numpy as np
+
+from validmind import tags, tasks
+from validmind.vm_models import VMDataset, VMModel
+
+
+@tasks("classification")
+@tags("classification")
+def CalibrationError(
+    model: VMModel, dataset: VMDataset, n_bins: int = 10, **kwargs
+) -> List[float]:
+    """Calculates the calibration error per row for a classification model.
+
+    Calibration error measures how well the predicted probabilities reflect the
+    actual likelihood of the positive class. For each prediction, this computes
+    the absolute difference between the predicted probability and the empirical
+    frequency of the positive class in the corresponding probability bin.
+
+    Args:
+        model: The classification model to evaluate
+        dataset: The dataset containing true labels and predicted probabilities
+        n_bins: Number of bins for probability calibration, defaults to 10
+        **kwargs: Additional parameters (unused for compatibility)
+
+    Returns:
+        List[float]: Per-row calibration errors as a list of float values
+
+    Raises:
+        ValueError: If probability column is not found for the model
+    """
+    y_true = dataset.y
+
+    # Try to get probabilities
+    try:
+        y_prob = dataset.y_prob(model)
+        # For binary classification, use the positive class probability
+        if y_prob.ndim > 1 and y_prob.shape[1] > 1:
+            y_prob = y_prob[:, 1]  # Use probability of positive class
+    except ValueError:
+        # If no probabilities available, return zeros (perfect calibration for hard predictions)
+        return [0.0] * len(y_true)
+
+    # Convert to numpy arrays
+    y_true = np.asarray(y_true, dtype=float)
+    y_prob = np.asarray(y_prob, dtype=float)
+
+    # Create probability bins
+    bin_boundaries = np.linspace(0, 1, n_bins + 1)
+    bin_lowers = bin_boundaries[:-1]
+    bin_uppers = bin_boundaries[1:]
+
+    # Calculate calibration error for each sample
+    calibration_errors = np.zeros_like(y_prob)
+
+    for bin_lower, bin_upper in zip(bin_lowers, bin_uppers):
+        # Find samples in this bin
+        in_bin = (y_prob > bin_lower) & (y_prob <= bin_upper)
+        if not np.any(in_bin):
+            continue
+
+        # Calculate empirical frequency for this bin
+        empirical_freq = np.mean(y_true[in_bin])
+
+        # Calculate average predicted probability for this bin
+        avg_predicted_prob = np.mean(y_prob[in_bin])
+
+        # Assign calibration error to all samples in this bin
+        calibration_errors[in_bin] = abs(avg_predicted_prob - empirical_freq)
+
+    # Return as a list of floats
+    return calibration_errors.tolist()