trustcv 1.0.0__py3-none-any.whl

trustcv/__init__.py

@@ -0,0 +1,300 @@
+"""
+trustcv - Trustworthy Cross-Validation Toolkit
+
+Developed at SMAILE (Stockholm Medical Artificial Intelligence and Learning Environments)
+Karolinska Institutet Core Facility - https://smile.ki.se
+Contributors: Farhad Abtahi, Abdelamir Karbalaie, SMAILE Team
+=============================================
+
+A framework-agnostic Python library for trustworthy cross-validation that provides a unified 
+interface across scikit-learn, PyTorch, TensorFlow/Keras, MONAI, JAX, and other 
+ML frameworks. It promotes best practices in model evaluation, providing both 
+familiar interfaces and customizable validators tailored for reliable validation.
+
+Developed at SMAILE (Stockholm Medical Artificial Intelligence and Learning Environments)
+Karolinska Institutet Core Facility
+Website: https://smile.ki.se
+
+Main Features:
+- Framework-agnostic: Works with any ML/DL framework
+- Advanced cross-validation strategies (hierarchical, temporal, spatial)
+- Automatic data leakage detection
+- Patient/group-level and temporal splitting
+- Regulatory compliance reporting (FDA, CE MDR)
+- Interactive visualizations
+- Support for PyTorch, TensorFlow, MONAI, and more
+
+Contributors:
+- Farhad Abtahi
+- Abdelamir Karbalaie
+- SMAILE Team
+
+For more information about AI research and collaboration opportunities,
+visit SMAILE at https://smile.ki.se
+"""
+
+__version__ = "1.0.0"
+__author__ = "SMAILE Team, Karolinska Institutet"
+__institution__ = "SMAILE - Stockholm Medical AI and Learning Environments, Karolinska Institutet"
+__website__ = "https://smile.ki.se"
+
+from .checkers import BalanceChecker, DataLeakageChecker
+
+# Import new framework-agnostic components
+from .core import (
+    ClassDistributionLogger,
+    CVCallback,
+    CVResults,
+    EarlyStopping,
+    ModelCheckpoint,
+    ProgressLogger,
+    UniversalCVRunner,
+)
+from .metrics import ClinicalMetrics
+
+# Import all splitters
+from .splitters import (  # I.I.D. methods; Grouped methods; Temporal methods; Spatial methods
+    LOOCV,
+    LPOCV,
+    BlockedTimeSeries,
+    BootstrapValidation,
+    BufferedSpatialCV,
+    CombinatorialPurgedCV,
+    EnvironmentalHealthCV,
+    ExpandingWindowCV,
+    GroupKFoldMedical,
+    HierarchicalGroupKFold,
+    HoldOut,
+    KFoldMedical,
+    LeaveOneGroupOut,
+    MonteCarloCV,
+    NestedCV,
+    NestedGroupedCV,
+    NestedTemporalCV,
+    PurgedGroupTimeSeriesSplit,
+    PurgedKFoldCV,
+    RepeatedGroupKFold,
+    RepeatedKFold,
+    RollingWindowCV,
+    SpatialBlockCV,
+    SpatiotemporalBlockCV,
+    StratifiedGroupKFold,
+    StratifiedKFoldMedical,
+    TimeSeriesSplit,
+)
+from .validators import MedicalValidator, NestedTemporalCV, TrustCVValidator
+
+# Conditionally import framework-specific runners if available
+try:
+    from .frameworks.pytorch import TorchCVRunner
+
+    _pytorch_available = True
+except ImportError:
+    _pytorch_available = False
+
+try:
+    from .frameworks.tensorflow import KerasCVRunner
+
+    _tensorflow_available = True
+except ImportError:
+    _tensorflow_available = False
+
+try:
+    from .frameworks.monai import MONAICVRunner
+
+    _monai_available = True
+except ImportError:
+    _monai_available = False
+
+__all__ = [
+    # Core validators and checkers
+    "TrustCVValidator",
+    "MedicalValidator",
+    "DataLeakageChecker",
+    "BalanceChecker",
+    "ClinicalMetrics",
+    "NestedTemporalCV",
+    "NestedGroupedCV",
+    # Framework-agnostic components
+    "UniversalCVRunner",
+    "CVResults",
+    "CVCallback",
+    "EarlyStopping",
+    "ModelCheckpoint",
+    "ProgressLogger",
+    "ClassDistributionLogger",
+    # I.I.D. methods
+    "HoldOut",
+    "KFoldMedical",
+    "StratifiedKFoldMedical",
+    "RepeatedKFold",
+    "LOOCV",
+    "LPOCV",
+    "BootstrapValidation",
+    "MonteCarloCV",
+    "NestedCV",
+    # Grouped methods
+    "GroupKFoldMedical",
+    "StratifiedGroupKFold",
+    "LeaveOneGroupOut",
+    "RepeatedGroupKFold",
+    "NestedGroupedCV",
+    "HierarchicalGroupKFold",
+    # Temporal methods
+    "TimeSeriesSplit",
+    "BlockedTimeSeries",
+    "RollingWindowCV",
+    "ExpandingWindowCV",
+    "PurgedKFoldCV",
+    "CombinatorialPurgedCV",
+    "PurgedGroupTimeSeriesSplit",
+    "NestedTemporalCV",
+    # Spatial methods
+    "SpatialBlockCV",
+    "BufferedSpatialCV",
+    "SpatiotemporalBlockCV",
+    "EnvironmentalHealthCV",
+]
+
+# Add framework-specific runners if available
+if _pytorch_available:
+    __all__.append("TorchCVRunner")
+if _tensorflow_available:
+    __all__.append("KerasCVRunner")
+if _monai_available:
+    __all__.append("MONAICVRunner")
+
+# === Canonical name aliases (consistency with sklearn/literature) ===
+import warnings as _trustcv_warnings
+
+
+def _trustcv_deprecated(old, new):
+    _trustcv_warnings.warn(
+        f"'{old}' is deprecated; use '{new}' instead.", category=DeprecationWarning, stacklevel=2
+    )
+
+
+# ---- Canonical aliases (bind canonical names to existing implementations) ----
+# IID / Grouped
+try:
+    KFold = KFoldMedical
+except NameError:
+    pass
+try:
+    StratifiedKFold = StratifiedKFoldMedical
+except NameError:
+    pass
+try:
+    LeaveOneOut = LOOCV
+except NameError:
+    pass
+try:
+    LeavePOut = LPOCV
+except NameError:
+    pass
+try:
+    GroupKFold = GroupKFoldMedical
+except NameError:
+    pass
+
+# Temporal
+try:
+    BlockedTimeSeriesSplit = BlockedTimeSeries
+except NameError:
+    pass
+try:
+    RollingWindowSplit = RollingWindowCV
+except NameError:
+    pass
+try:
+    ExpandingWindowSplit = ExpandingWindowCV
+except NameError:
+    pass
+try:
+    PurgedKFold = PurgedKFoldCV
+except NameError:
+    pass
+try:
+    CombinatorialPurgedKFold = CombinatorialPurgedCV
+except NameError:
+    pass
+
+# Spatial
+try:
+    SpatialBlockSplit = SpatialBlockCV
+except NameError:
+    pass
+try:
+    BufferedSpatialSplit = BufferedSpatialCV
+except NameError:
+    pass
+try:
+    SpatiotemporalBlockSplit = SpatiotemporalBlockCV
+except NameError:
+    pass
+try:
+    EnvironmentalHealthSplit = EnvironmentalHealthCV
+except NameError:
+    pass
+
+# Checkers
+try:
+    LeakageChecker = DataLeakageChecker
+except NameError:
+    pass
+try:
+    ClassBalanceChecker = BalanceChecker
+except NameError:
+    pass
+
+# ---- Extend __all__ with canonical names (non-breaking) ----
+__all__.extend(
+    [
+        # IID / Grouped
+        "KFold",
+        "StratifiedKFold",
+        "LeaveOneOut",
+        "LeavePOut",
+        "GroupKFold",
+        # Temporal
+        "BlockedTimeSeriesSplit",
+        "RollingWindowSplit",
+        "ExpandingWindowSplit",
+        "PurgedKFold",
+        "CombinatorialPurgedKFold",
+        # Spatial
+        "SpatialBlockSplit",
+        "BufferedSpatialSplit",
+        "SpatiotemporalBlockSplit",
+        "EnvironmentalHealthSplit",
+        # Checkers
+        "LeakageChecker",
+        "ClassBalanceChecker",
+    ]
+)
+
+
+# ---- Optional: gentle deprecation notices (future-proofing) ----
+def __getattr__(name):
+    _map = {
+        "KFoldMedical": "KFold",
+        "StratifiedKFoldMedical": "StratifiedKFold",
+        "LOOCV": "LeaveOneOut",
+        "LPOCV": "LeavePOut",
+        "GroupKFoldMedical": "GroupKFold",
+        "BlockedTimeSeries": "BlockedTimeSeriesSplit",
+        "RollingWindowCV": "RollingWindowSplit",
+        "ExpandingWindowCV": "ExpandingWindowSplit",
+        "PurgedKFoldCV": "PurgedKFold",
+        "CombinatorialPurgedCV": "CombinatorialPurgedKFold",
+        "SpatialBlockCV": "SpatialBlockSplit",
+        "BufferedSpatialCV": "BufferedSpatialSplit",
+        "SpatiotemporalBlockCV": "SpatiotemporalBlockSplit",
+        "EnvironmentalHealthCV": "EnvironmentalHealthSplit",
+        "DataLeakageChecker": "LeakageChecker",
+        "BalanceChecker": "ClassBalanceChecker",
+    }
+    if name in _map:
+        _trustcv_deprecated(name, _map[name])
+        return globals().get(_map[name])
+    raise AttributeError(f"module 'trustcv' has no attribute '{name}'")

trustcv/checkers/__init__.py

@@ -0,0 +1,12 @@
+"""
+Data integrity and leakage checkers for ML
+
+Developed at SMAILE (Stockholm Medical Artificial Intelligence and Learning Environments)
+Karolinska Institutet Core Facility - https://smile.ki.se
+Contributors: Farhad Abtahi, Abdelamir Karbalaie, SMAILE Team
+"""
+
+from .balance import BalanceChecker
+from .leakage import DataLeakageChecker
+
+__all__ = ["DataLeakageChecker", "BalanceChecker"]

trustcv/checkers/balance.py

@@ -0,0 +1,353 @@
+"""
+Balance checker for medical datasets
+
+Developed at SMAILE (Stockholm Medical Artificial Intelligence and Learning Environments)
+Karolinska Institutet Core Facility - https://smile.ki.se
+Contributors: Farhad Abtahi, Abdelamir Karbalaie, SMAILE Team
+"""
+
+import warnings
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+
+
+class BalanceChecker:
+    """
+    Checks for class imbalance and data distribution issues
+
+    Critical for medical datasets where disease prevalence
+    can be extremely low (< 1% positive cases)
+    """
+
+    def __init__(self, threshold: float = 0.1):
+        """
+        Parameters
+        ----------
+        threshold : float
+            Imbalance threshold for warnings (default 10% difference)
+        """
+        self.threshold = threshold
+        self.report = {}
+
+    def check_class_balance(
+        self, y: np.ndarray, groups: Optional[np.ndarray] = None
+    ) -> Dict[str, Any]:
+        """
+        Check class distribution in dataset
+
+        Parameters
+        ----------
+        y : array-like
+            Target labels
+        groups : array-like, optional
+            Group identifiers (e.g., patient IDs)
+
+        Returns
+        -------
+        dict
+            Balance report with statistics and warnings
+        """
+        y = np.array(y)
+        unique_classes, counts = np.unique(y, return_counts=True)
+        n_samples = len(y)
+
+        # Calculate class distribution
+        class_dist = {
+            str(cls): {"count": int(count), "percentage": float(count / n_samples * 100)}
+            for cls, count in zip(unique_classes, counts)
+        }
+
+        # Calculate imbalance ratio
+        min_class = counts.min()
+        max_class = counts.max()
+        imbalance_ratio = max_class / min_class if min_class > 0 else np.inf
+
+        # Check for severe imbalance
+        minority_pct = min_class / n_samples
+
+        report = {
+            "n_classes": len(unique_classes),
+            "class_distribution": class_dist,
+            "imbalance_ratio": float(imbalance_ratio),
+            "minority_percentage": float(minority_pct * 100),
+            "warnings": [],
+        }
+
+        # Generate warnings
+        if imbalance_ratio > 10:
+            report["warnings"].append(
+                f"SEVERE IMBALANCE: Ratio {imbalance_ratio:.1f}:1. "
+                "Consider stratified splitting or resampling."
+            )
+        elif imbalance_ratio > 5:
+            report["warnings"].append(
+                f"Moderate imbalance: Ratio {imbalance_ratio:.1f}:1. "
+                "Use stratified cross-validation."
+            )
+
+        if minority_pct < 0.01:
+            report["warnings"].append(
+                f"Rare event: Minority class < 1% ({minority_pct*100:.2f}%). "
+                "Consider specialized techniques (SMOTE, cost-sensitive learning)."
+            )
+
+        # Check group-level balance if provided
+        if groups is not None:
+            report["group_analysis"] = self._check_group_balance(y, groups)
+
+        self.report = report
+        return report
+
+    def _check_group_balance(self, y: np.ndarray, groups: np.ndarray) -> Dict[str, Any]:
+        """
+        Check balance within groups (e.g., per patient)
+
+        Parameters
+        ----------
+        y : array-like
+            Target labels
+        groups : array-like
+            Group identifiers
+
+        Returns
+        -------
+        dict
+            Group-level balance analysis
+        """
+        groups = np.array(groups)
+        unique_groups = np.unique(groups)
+
+        group_stats = []
+        for group in unique_groups:
+            group_mask = groups == group
+            group_y = y[group_mask]
+
+            if len(np.unique(group_y)) > 1:
+                # Mixed labels in group
+                unique_classes, counts = np.unique(group_y, return_counts=True)
+                majority_class = unique_classes[np.argmax(counts)]
+                group_stats.append(
+                    {
+                        "group": group,
+                        "n_samples": len(group_y),
+                        "n_classes": len(unique_classes),
+                        "majority_class": int(majority_class),
+                        "mixed": True,
+                    }
+                )
+            else:
+                # Single class in group
+                group_stats.append(
+                    {
+                        "group": group,
+                        "n_samples": len(group_y),
+                        "n_classes": 1,
+                        "majority_class": int(group_y[0]),
+                        "mixed": False,
+                    }
+                )
+
+        # Analyze group patterns
+        mixed_groups = sum(1 for g in group_stats if g["mixed"])
+        single_class_groups = len(group_stats) - mixed_groups
+
+        analysis = {
+            "n_groups": len(unique_groups),
+            "mixed_label_groups": mixed_groups,
+            "single_label_groups": single_class_groups,
+            "warnings": [],
+        }
+
+        if mixed_groups > 0:
+            analysis["warnings"].append(
+                f"{mixed_groups} groups have mixed labels. "
+                "This may affect patient-level predictions."
+            )
+
+        return analysis
+
+    def check_feature_distribution(
+        self, X: np.ndarray, feature_names: Optional[List[str]] = None
+    ) -> Dict[str, Any]:
+        """
+        Check feature distributions for potential issues
+
+        Parameters
+        ----------
+        X : array-like
+            Feature matrix
+        feature_names : list, optional
+            Names of features
+
+        Returns
+        -------
+        dict
+            Feature distribution analysis
+        """
+        X = np.array(X)
+        n_samples, n_features = X.shape
+
+        if feature_names is None:
+            feature_names = [f"Feature_{i}" for i in range(n_features)]
+
+        feature_stats = []
+        warnings = []
+
+        for i, name in enumerate(feature_names):
+            feature = X[:, i]
+
+            stats = {
+                "name": name,
+                "mean": float(np.mean(feature)),
+                "std": float(np.std(feature)),
+                "min": float(np.min(feature)),
+                "max": float(np.max(feature)),
+                "n_unique": len(np.unique(feature)),
+                "n_missing": int(np.isnan(feature).sum()),
+                "skewness": float(self._calculate_skewness(feature)),
+            }
+
+            # Check for issues
+            if stats["std"] == 0:
+                warnings.append(f"{name}: Zero variance (constant feature)")
+            elif stats["n_unique"] == 1:
+                warnings.append(f"{name}: Only one unique value")
+            elif stats["n_missing"] > n_samples * 0.2:
+                warnings.append(
+                    f"{name}: High missing rate ({stats['n_missing']/n_samples*100:.1f}%)"
+                )
+            elif abs(stats["skewness"]) > 2:
+                warnings.append(f"{name}: Highly skewed (skewness={stats['skewness']:.2f})")
+
+            feature_stats.append(stats)
+
+        return {"n_features": n_features, "feature_statistics": feature_stats, "warnings": warnings}
+
+    def _calculate_skewness(self, x: np.ndarray) -> float:
+        """Calculate skewness of a distribution"""
+        x = x[~np.isnan(x)]  # Remove NaN values
+        if len(x) < 3:
+            return 0.0
+
+        mean = np.mean(x)
+        std = np.std(x)
+
+        if std == 0:
+            return 0.0
+
+        skewness = np.mean(((x - mean) / std) ** 3)
+        return skewness
+
+    def check_cv_balance(self, X, y, cv_splitter, groups=None):
+        """
+        Check balance across CV folds
+
+        Parameters
+        ----------
+        X : array-like
+            Features
+        y : array-like
+            Labels
+        cv_splitter : cross-validation object
+            CV splitter to test
+        groups : array-like, optional
+            Group labels
+
+        Returns
+        -------
+        dict
+            CV balance analysis
+        """
+        y = np.array(y)
+        fold_stats = []
+
+        for fold_idx, (train_idx, test_idx) in enumerate(cv_splitter.split(X, y, groups)):
+            y_train = y[train_idx]
+            y_test = y[test_idx]
+
+            # Calculate distributions
+            train_dist = np.bincount(y_train) / len(y_train)
+            test_dist = np.bincount(y_test) / len(y_test)
+
+            # Ensure same length
+            max_len = max(len(train_dist), len(test_dist))
+            train_dist = np.pad(train_dist, (0, max_len - len(train_dist)))
+            test_dist = np.pad(test_dist, (0, max_len - len(test_dist)))
+
+            # Calculate distribution difference
+            dist_diff = np.abs(train_dist - test_dist).max()
+
+            fold_stats.append(
+                {
+                    "fold": fold_idx,
+                    "train_size": len(train_idx),
+                    "test_size": len(test_idx),
+                    "train_distribution": train_dist.tolist(),
+                    "test_distribution": test_dist.tolist(),
+                    "max_distribution_diff": float(dist_diff),
+                }
+            )
+
+        # Check for issues
+        warnings = []
+        max_diff = max(f["max_distribution_diff"] for f in fold_stats)
+
+        if max_diff > self.threshold:
+            warnings.append(
+                f"Large distribution difference between train/test: {max_diff*100:.1f}%. "
+                "Consider stratified splitting."
+            )
+
+        size_variation = np.std([f["test_size"] for f in fold_stats])
+        if size_variation > np.mean([f["test_size"] for f in fold_stats]) * 0.1:
+            warnings.append(
+                "Large variation in fold sizes. " "This may affect performance estimates."
+            )
+
+        return {
+            "n_folds": len(fold_stats),
+            "fold_statistics": fold_stats,
+            "max_distribution_difference": float(max_diff),
+            "warnings": warnings,
+        }
+
+    def generate_report(self) -> str:
+        """
+        Generate human-readable balance report
+
+        Returns
+        -------
+        str
+            Formatted report
+        """
+        if not self.report:
+            return "No balance check performed yet."
+
+        lines = ["=" * 50, "BALANCE CHECK REPORT", "=" * 50]
+
+        # Class balance
+        if "class_distribution" in self.report:
+            lines.append("\nCLASS BALANCE:")
+            for cls, stats in self.report["class_distribution"].items():
+                lines.append(
+                    f"  Class {cls}: {stats['count']} samples ({stats['percentage']:.1f}%)"
+                )
+            lines.append(f"  Imbalance ratio: {self.report['imbalance_ratio']:.2f}:1")
+
+        # Warnings
+        if self.report.get("warnings"):
+            lines.append("\nWARNINGS:")
+            for warning in self.report["warnings"]:
+                lines.append(f"  ⚠️  {warning}")
+
+        # Group analysis
+        if "group_analysis" in self.report:
+            ga = self.report["group_analysis"]
+            lines.append(f"\nGROUP ANALYSIS:")
+            lines.append(f"  Total groups: {ga['n_groups']}")
+            lines.append(f"  Mixed-label groups: {ga['mixed_label_groups']}")
+            lines.append(f"  Single-label groups: {ga['single_label_groups']}")
+
+        return "\n".join(lines)