nestkit 0.2.0__tar.gz → 0.3.0__tar.gz

{nestkit-0.2.0/src/nestkit.egg-info → nestkit-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nestkit
-Version: 0.2.0
+Version: 0.3.0
 Summary: Nested cross-validation toolkit with optional probability calibration, threshold optimization, and comprehensive diagnostics
 Author: Ettore Rocchi
 License: MIT
@@ -22,14 +22,16 @@ Requires-Dist: pandas>=1.4
 Requires-Dist: joblib>=1.2
 Requires-Dist: scipy>=1.9
 Requires-Dist: jinja2>=3.0
-Provides-Extra: plotting
-Requires-Dist: matplotlib>=3.5; extra == "plotting"
-Requires-Dist: seaborn>=0.12; extra == "plotting"
+Requires-Dist: matplotlib>=3.10
+Requires-Dist: seaborn>=0.12
 Provides-Extra: bayesian
 Requires-Dist: scikit-optimize>=0.9; extra == "bayesian"
+Provides-Extra: survival
+Requires-Dist: lifelines>=0.27; extra == "survival"
+Provides-Extra: explainability
+Requires-Dist: shap>=0.42; extra == "explainability"
 Provides-Extra: full
-Requires-Dist: nestkit[bayesian,plotting]; extra == "full"
-Requires-Dist: shap>=0.42; extra == "full"
+Requires-Dist: nestkit[bayesian,explainability,survival]; extra == "full"
 Provides-Extra: dev
 Requires-Dist: pytest>=7; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
@@ -44,6 +46,7 @@ Requires-Dist: numpydoc; extra == "docs"
 Requires-Dist: nbsphinx; extra == "docs"
 Requires-Dist: ipykernel; extra == "docs"
 Requires-Dist: sphinx-design; extra == "docs"
+Requires-Dist: myst-parser; extra == "docs"
 Dynamic: license-file
 
 <p align="center">
@@ -73,6 +76,7 @@ Standard cross-validation conflates model selection with performance estimation,
 ## Key Features
 
 - **Nested cross-validation** for classification and regression with full scikit-learn API compatibility
+- **Missing values**  -  `X` may contain `NaN`; an imputer placed in a `Pipeline` is refitted inside every inner and outer fold, so imputation is leakage-free
 - **Post-hoc probability calibration**  -  Platt scaling, isotonic regression, beta calibration, and Venn-ABERS prediction
 - **Threshold optimization**  -  Youden's J, F-beta, cost-sensitive, balanced accuracy, and precision-at-recall criteria with pooled or fold-specific strategies
 - **CV+ Mondrian conformal prediction**  -  class-conditional prediction sets (classification) and Mondrian-binned conditional prediction intervals (regression) with formal coverage guarantees
@@ -82,6 +86,7 @@ Standard cross-validation conflates model selection with performance estimation,
 - **Callback system**  -  progress tracking, logging, checkpointing, and custom hooks
 - **25+ plotting functions**  -  ROC curves, confusion matrices, calibration diagrams, threshold sensitivity, critical difference diagrams, and more
 - **Prediction intervals** for regression  -  global or Mondrian-binned conformal intervals
+- **Survival analysis**  -  nested CV for Cox PH models via lifelines with Harrell's and Uno's concordance indices, integrated Brier score, and coefficient stability analysis
 
 ## Installation
 
@@ -92,10 +97,12 @@ pip install nestkit
 Optional dependency groups:
 
 ```bash
-pip install nestkit[plotting]   # matplotlib + seaborn
-pip install nestkit[full]       # plotting + SHAP
-pip install nestkit[dev]        # testing + linting
-pip install nestkit[docs]       # Sphinx documentation
+pip install nestkit[survival]       # lifelines (survival analysis)
+pip install nestkit[bayesian]       # scikit-optimize (Bayesian search)
+pip install nestkit[explainability] # SHAP
+pip install nestkit[full]           # all of the above
+pip install nestkit[dev]            # testing + linting
+pip install nestkit[docs]           # Sphinx documentation
 ```
 
 ## Quick Start
@@ -158,6 +165,50 @@ ncv.fit(X, y)
 print(ncv.results_.conformal_report())
 ```
 
+With missing values (the imputer is fitted within each fold):
+
+```python
+from sklearn.impute import SimpleImputer
+from sklearn.pipeline import make_pipeline
+
+ncv = NestedCVClassifier(
+    estimator=make_pipeline(
+        SimpleImputer(keep_empty_features=True),
+        RandomForestClassifier(random_state=42),
+    ),
+    param_grid={"randomforestclassifier__max_depth": [3, 5, 10]},
+    outer_cv=5,
+    inner_cv=3,
+    random_state=42,
+)
+ncv.fit(X, y)  # X may contain NaN
+```
+
+### Survival Analysis
+
+```python
+from nestkit import NestedCVSurvival
+from nestkit.survival import CoxPHWrapper, make_survival_target
+from lifelines.datasets import load_rossi
+
+rossi = load_rossi()
+X = rossi.drop(columns=["week", "arrest"])
+y = make_survival_target(event=rossi["arrest"].values, duration=rossi["week"].values)
+
+ncv = NestedCVSurvival(
+    estimator=CoxPHWrapper(),
+    param_grid={"penalizer": [0.001, 0.01, 0.1, 1.0], "l1_ratio": [0.0, 0.5, 1.0]},
+    outer_cv=5,
+    inner_cv=3,
+    random_state=42,
+)
+ncv.fit(X, y)
+
+results = ncv.results_
+print(results.summary_default_)          # Harrell C-index, Uno C-index, IBS
+print(results.coefficient_stability_)    # hazard ratio stability across folds
+```
+
 ### Regression
 
 ```python
@@ -197,7 +248,9 @@ nestkit's nested CV procedure executes four phases per outer fold:
 |-------|---------|
 | `NestedCVClassifier` | Classification with calibration, thresholding, and conformal prediction |
 | `NestedCVRegressor` | Regression with prediction intervals and Mondrian binning |
-| `ClassifierResults` / `RegressorResults` | Rich result containers |
+| `NestedCVSurvival` | Survival analysis with Cox PH models (requires lifelines) |
+| `CoxPHWrapper` | sklearn-compatible wrapper for lifelines' `CoxPHFitter` |
+| `ClassifierResults` / `RegressorResults` / `SurvivalResults` | Rich result containers |
 | `NestedCVComparator` | Statistical model comparison |
 | `FeatureImportanceAggregator` | Cross-fold importance analysis |
 | `HyperparameterStability` | Selection consistency diagnostics |

{nestkit-0.2.0 → nestkit-0.3.0}/README.md

@@ -25,6 +25,7 @@ Standard cross-validation conflates model selection with performance estimation,
 ## Key Features
 
 - **Nested cross-validation** for classification and regression with full scikit-learn API compatibility
+- **Missing values**  -  `X` may contain `NaN`; an imputer placed in a `Pipeline` is refitted inside every inner and outer fold, so imputation is leakage-free
 - **Post-hoc probability calibration**  -  Platt scaling, isotonic regression, beta calibration, and Venn-ABERS prediction
 - **Threshold optimization**  -  Youden's J, F-beta, cost-sensitive, balanced accuracy, and precision-at-recall criteria with pooled or fold-specific strategies
 - **CV+ Mondrian conformal prediction**  -  class-conditional prediction sets (classification) and Mondrian-binned conditional prediction intervals (regression) with formal coverage guarantees
@@ -34,6 +35,7 @@ Standard cross-validation conflates model selection with performance estimation,
 - **Callback system**  -  progress tracking, logging, checkpointing, and custom hooks
 - **25+ plotting functions**  -  ROC curves, confusion matrices, calibration diagrams, threshold sensitivity, critical difference diagrams, and more
 - **Prediction intervals** for regression  -  global or Mondrian-binned conformal intervals
+- **Survival analysis**  -  nested CV for Cox PH models via lifelines with Harrell's and Uno's concordance indices, integrated Brier score, and coefficient stability analysis
 
 ## Installation
 
@@ -44,10 +46,12 @@ pip install nestkit
 Optional dependency groups:
 
 ```bash
-pip install nestkit[plotting]   # matplotlib + seaborn
-pip install nestkit[full]       # plotting + SHAP
-pip install nestkit[dev]        # testing + linting
-pip install nestkit[docs]       # Sphinx documentation
+pip install nestkit[survival]       # lifelines (survival analysis)
+pip install nestkit[bayesian]       # scikit-optimize (Bayesian search)
+pip install nestkit[explainability] # SHAP
+pip install nestkit[full]           # all of the above
+pip install nestkit[dev]            # testing + linting
+pip install nestkit[docs]           # Sphinx documentation
 ```
 
 ## Quick Start
@@ -110,6 +114,50 @@ ncv.fit(X, y)
 print(ncv.results_.conformal_report())
 ```
 
+With missing values (the imputer is fitted within each fold):
+
+```python
+from sklearn.impute import SimpleImputer
+from sklearn.pipeline import make_pipeline
+
+ncv = NestedCVClassifier(
+    estimator=make_pipeline(
+        SimpleImputer(keep_empty_features=True),
+        RandomForestClassifier(random_state=42),
+    ),
+    param_grid={"randomforestclassifier__max_depth": [3, 5, 10]},
+    outer_cv=5,
+    inner_cv=3,
+    random_state=42,
+)
+ncv.fit(X, y)  # X may contain NaN
+```
+
+### Survival Analysis
+
+```python
+from nestkit import NestedCVSurvival
+from nestkit.survival import CoxPHWrapper, make_survival_target
+from lifelines.datasets import load_rossi
+
+rossi = load_rossi()
+X = rossi.drop(columns=["week", "arrest"])
+y = make_survival_target(event=rossi["arrest"].values, duration=rossi["week"].values)
+
+ncv = NestedCVSurvival(
+    estimator=CoxPHWrapper(),
+    param_grid={"penalizer": [0.001, 0.01, 0.1, 1.0], "l1_ratio": [0.0, 0.5, 1.0]},
+    outer_cv=5,
+    inner_cv=3,
+    random_state=42,
+)
+ncv.fit(X, y)
+
+results = ncv.results_
+print(results.summary_default_)          # Harrell C-index, Uno C-index, IBS
+print(results.coefficient_stability_)    # hazard ratio stability across folds
+```
+
 ### Regression
 
 ```python
@@ -149,7 +197,9 @@ nestkit's nested CV procedure executes four phases per outer fold:
 |-------|---------|
 | `NestedCVClassifier` | Classification with calibration, thresholding, and conformal prediction |
 | `NestedCVRegressor` | Regression with prediction intervals and Mondrian binning |
-| `ClassifierResults` / `RegressorResults` | Rich result containers |
+| `NestedCVSurvival` | Survival analysis with Cox PH models (requires lifelines) |
+| `CoxPHWrapper` | sklearn-compatible wrapper for lifelines' `CoxPHFitter` |
+| `ClassifierResults` / `RegressorResults` / `SurvivalResults` | Rich result containers |
 | `NestedCVComparator` | Statistical model comparison |
 | `FeatureImportanceAggregator` | Cross-fold importance analysis |
 | `HyperparameterStability` | Selection consistency diagnostics |

{nestkit-0.2.0 → nestkit-0.3.0}/pyproject.toml

@@ -23,9 +23,10 @@ classifiers = [
 ]
 
 [project.optional-dependencies]
-plotting = ["matplotlib>=3.5", "seaborn>=0.12"]
 bayesian = ["scikit-optimize>=0.9"]
-full = ["nestkit[plotting,bayesian]", "shap>=0.42"]
+survival = ["lifelines>=0.27"]
+explainability = ["shap>=0.42"]
+full = ["nestkit[bayesian,survival,explainability]"]
 dev = [
     "pytest>=7",
     "pytest-cov",
@@ -41,6 +42,7 @@ docs = [
     "nbsphinx",
     "ipykernel",
     "sphinx-design",
+    "myst-parser",
 ]
 
 [tool.setuptools.dynamic]
@@ -76,8 +78,10 @@ ignore = [
 known-first-party = ["nestkit"]
 
 [tool.pytest.ini_options]
-testpaths = ["tests"]
-addopts = "-ra -q"
+# src is included so that --doctest-modules actually runs the docstring
+# examples; without it pytest would only collect from tests/.
+testpaths = ["tests", "src"]
+addopts = "-ra -q --doctest-modules"
 markers = [
     "slow: end-to-end tests that fit NestedCV estimators (deselect with '-m \"not slow\"')",
 ]

{nestkit-0.2.0 → nestkit-0.3.0}/requirements.txt

@@ -4,3 +4,5 @@ pandas>=1.4
 joblib>=1.2
 scipy>=1.9
 jinja2>=3.0
+matplotlib>=3.10
+seaborn>=0.12

{nestkit-0.2.0 → nestkit-0.3.0}/src/nestkit/__init__.py

@@ -26,6 +26,8 @@ Submodules
 - :mod:`nestkit.conformal`  -  CV+ Mondrian conformal prediction sets and
   intervals.
 - :mod:`nestkit.thresholding`  -  Decision-threshold optimization criteria.
+- :mod:`nestkit.survival`  -  Survival analysis with Cox PH models
+  (requires lifelines).
 """
 
 from __future__ import annotations
@@ -36,16 +38,22 @@ from nestkit.classifier import NestedCVClassifier
 from nestkit.regressor import NestedCVRegressor
 from nestkit.results.classifier_results import ClassifierResults
 from nestkit.results.regressor_results import RegressorResults
+from nestkit.results.survival_results import SurvivalResults
+from nestkit.survival import CoxPHWrapper, NestedCVSurvival, make_survival_target
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 
 logger = logging.getLogger("nestkit")
 logger.addHandler(logging.NullHandler())
 
 __all__ = [
     "ClassifierResults",
+    "CoxPHWrapper",
     "NestedCVClassifier",
     "NestedCVRegressor",
+    "NestedCVSurvival",
     "RegressorResults",
+    "SurvivalResults",
     "__version__",
+    "make_survival_target",
 ]

{nestkit-0.2.0 → nestkit-0.3.0}/src/nestkit/_base.py

@@ -8,6 +8,7 @@ aggregation.
 
 from __future__ import annotations
 
+import inspect
 import logging
 import time
 from abc import ABCMeta, abstractmethod
@@ -22,6 +23,13 @@ from nestkit.inner.search import build_search
 
 logger = logging.getLogger("nestkit")
 
+# ``force_all_finite`` was renamed to ``ensure_all_finite`` in scikit-learn 1.6
+_ALLOW_NAN = (
+    {"ensure_all_finite": "allow-nan"}
+    if "ensure_all_finite" in inspect.signature(check_X_y).parameters
+    else {"force_all_finite": "allow-nan"}
+)
+
 
 class _BaseNestedCV(BaseEstimator, metaclass=ABCMeta):
     """Abstract base class for nested cross-validation estimators.
@@ -176,7 +184,12 @@ class _BaseNestedCV(BaseEstimator, metaclass=ABCMeta):
         ----------
         X : array-like of shape (n_samples, n_features)
             Training data. If a pandas DataFrame is passed, feature names
-            and the original index are preserved in the results.
+            and the original index are preserved in the results. Missing
+            values encoded as ``NaN`` (or ``pd.NA``) are allowed, so that
+            imputation can be fitted inside each fold by making the
+            imputer the first step of a
+            :class:`~sklearn.pipeline.Pipeline` estimator. Infinite
+            values are rejected.
         y : array-like of shape (n_samples,)
             Target values.
         groups : array-like of shape (n_samples,) or None, default=None
@@ -195,18 +208,22 @@ class _BaseNestedCV(BaseEstimator, metaclass=ABCMeta):
         Raises
         ------
         ValueError
-            If ``X`` and ``y`` have incompatible shapes.
+            If ``X`` and ``y`` have incompatible shapes, if ``X`` contains
+            infinite values, or if ``y`` contains NaN or infinite values.
         """
         # DataFrame handling
         if hasattr(X, "columns"):
             self.feature_names_in_ = list(X.columns)
             self._original_index = X.index.copy()
-            X = X.to_numpy()
         else:
             self.feature_names_in_ = [f"feature_{i}" for i in range(X.shape[1])]
             self._original_index = None
 
-        X, y = check_X_y(X, y, multi_output=True, allow_nd=True)
+        # NaN is allowed in X so that imputation can be fitted inside the
+        # folds (e.g. as the first step of a Pipeline); inf and NaN in y are
+        # still rejected. DataFrames are converted here so that pandas
+        # nullable dtypes (pd.NA) become NaN.
+        X, y = check_X_y(X, y, multi_output=True, allow_nd=True, **_ALLOW_NAN)
 
         outer_cv = check_cv(self.outer_cv, y, classifier=is_classifier(self.estimator))
         splits = list(outer_cv.split(X, y, groups))

{nestkit-0.2.0 → nestkit-0.3.0}/src/nestkit/_validation.py

@@ -123,6 +123,61 @@ def validate_mondrian_params(
         )
 
 
+def validate_survival_target(y: np.ndarray) -> None:
+    """Validate a normalized survival target array.
+
+    Parameters
+    ----------
+    y : ndarray of shape (n_samples, 2)
+        Survival target with columns ``[event, duration]``.
+        Must already be normalized via
+        :func:`~nestkit.survival._target._normalize_survival_target`.
+
+    Raises
+    ------
+    ValueError
+        If the target does not have the expected shape or content.
+
+    Warns
+    -----
+    UserWarning
+        If the censoring rate is extremely high (>95%) or
+        extremely low (<5%).
+    """
+    import warnings
+
+    if y.ndim != 2 or y.shape[1] != 2:
+        raise ValueError(f"Survival target must have shape (n_samples, 2), got {y.shape}")
+
+    event = y[:, 0]
+    duration = y[:, 1]
+
+    unique_events = np.unique(event)
+    if not np.all(np.isin(unique_events, [0.0, 1.0])):
+        raise ValueError(
+            f"Event column must be binary (0 or 1), got unique values {unique_events}"
+        )
+
+    if np.any(duration <= 0):
+        raise ValueError("Duration column must contain only positive values")
+
+    censoring_rate = 1.0 - np.mean(event)
+    if censoring_rate > 0.95:
+        warnings.warn(
+            f"Very high censoring rate ({censoring_rate:.1%}). "
+            "Model performance and metric reliability may be poor.",
+            UserWarning,
+            stacklevel=2,
+        )
+    elif censoring_rate < 0.05:
+        warnings.warn(
+            f"Very low censoring rate ({censoring_rate:.1%}). "
+            "Consider whether a standard regression model might be more appropriate.",
+            UserWarning,
+            stacklevel=2,
+        )
+
+
 def ensure_2d_proba(y_proba: np.ndarray) -> np.ndarray:
     """Ensure probability array is 2D ``(n_samples, n_classes)``.
 

{nestkit-0.2.0 → nestkit-0.3.0}/src/nestkit/calibration/calibrators.py

@@ -42,9 +42,26 @@ class PostHocCalibrator:
         y_proba : array of shape (n_samples,) or (n_samples, 2)
             Uncalibrated predicted probabilities.
         y_true : array of shape (n_samples,)
-            True binary labels.
+            True binary labels, encoded as 0 (negative) and 1 (positive).
+
+        Raises
+        ------
+        ValueError
+            If *y_true* holds labels other than 0 and 1.  The isotonic and
+            Venn-ABERS fits regress directly onto the label values, so any
+            other encoding silently yields probabilities outside ``[0, 1]``
+            that are then clipped (e.g. labels ``{1, 2}`` map every sample
+            to 1.0).
         """
         p = extract_positive_proba(y_proba)
+        y_true = np.asarray(y_true)
+        observed = np.unique(y_true)
+        if not np.all(np.isin(observed, [0, 1])):
+            raise ValueError(
+                "y_true must be encoded as 0 (negative) and 1 (positive), "
+                f"got labels {observed.tolist()}. Encode the target first, "
+                "e.g. with sklearn.preprocessing.LabelEncoder."
+            )
 
         if self.method == "sigmoid":
             self._fit_sigmoid(p, y_true)

{nestkit-0.2.0 → nestkit-0.3.0}/src/nestkit/classifier.py

@@ -253,7 +253,9 @@ class NestedCVClassifier(_BaseNestedCV):
         Parameters
         ----------
         X : array-like of shape (n_samples, n_features)
-            Training data.
+            Training data. May contain ``NaN`` if the estimator handles
+            them, e.g. a Pipeline starting with an imputer (see
+            :ref:`missing-values`).
         y : array-like of shape (n_samples,)
             Target labels.
         groups : array-like of shape (n_samples,) or None, default=None
@@ -281,20 +283,25 @@ class NestedCVClassifier(_BaseNestedCV):
             self.min_recall,
         )
         validate_conformal_params(self.conformal_prediction, self.conformal_alpha)
-        self.classes_ = np.unique(y)
+
+        self.classes_, y_encoded = np.unique(y, return_inverse=True)
         self.n_classes_ = len(self.classes_)
-        return super().fit(X, y, groups=groups, **fit_params)
+        return super().fit(X, np.asarray(y_encoded).ravel(), groups=groups, **fit_params)
+
+    def _decode(self, y_encoded) -> np.ndarray:
+        """Map encoded labels (0..n_classes-1) back to the original labels."""
+        return self.classes_[np.asarray(y_encoded).astype(int)]
 
     def _build_results_container(self) -> type:
         return ClassifierResults
 
     def _post_inner_processing(self, search, X_train, y_train, groups_train, **fit_params) -> dict:
-        """Phase 2 + Phase 3: calibration and threshold optimization.
+        """Calibration and threshold optimization.
 
         Note: The OOF loop uses ``search.best_params_`` which were selected
         using all of ``X_train``. The OOF validation folds therefore
         influenced hyperparameter selection. This is a widely accepted
-        approximation -- the alternative (triple-nested CV) is
+        approximation - the alternative (triple-nested CV) is
         computationally prohibitive for most practical use cases.
         """
         artifacts: dict[str, Any] = {
@@ -318,7 +325,6 @@ class NestedCVClassifier(_BaseNestedCV):
         ):
             return artifacts
 
-        # Slow path: collect inner OOF predictions (always refit)
         cal_cv = check_cv(self.calibration_cv or self.inner_cv, y_train, classifier=True)
         best_params = search.best_params_
         base_estimator = clone(self.estimator).set_params(**best_params)
@@ -340,7 +346,6 @@ class NestedCVClassifier(_BaseNestedCV):
         n_classes = oof_probas_all.shape[1] if oof_probas_all.ndim == 2 else 2
         is_binary = n_classes == 2
 
-        # --- Phase 2: Calibration ---
         if self.calibration_method is not None:
             if is_binary:
                 calibrator = PostHocCalibrator(method=self.calibration_method)
@@ -357,7 +362,7 @@ class NestedCVClassifier(_BaseNestedCV):
                 cal_probas_per_fold = [np.zeros_like(p) for p in oof_probas]
 
                 for c in range(n_classes):
-                    y_binary = (oof_y_all == self.classes_[c]).astype(int)
+                    y_binary = (oof_y_all == c).astype(int)
                     p_c = oof_probas_all[:, c]
                     cal_c = PostHocCalibrator(method=self.calibration_method)
                     cal_c.fit(p_c, y_binary)
@@ -382,7 +387,6 @@ class NestedCVClassifier(_BaseNestedCV):
 
         artifacts["oof_probas_calibrated"] = cal_probas_all
 
-        # --- Phase 3: Threshold optimization ---
         if self.threshold_strategy is not None:
             criterion_fn = self._resolve_criterion()
             criterion_name = (
@@ -407,7 +411,7 @@ class NestedCVClassifier(_BaseNestedCV):
                 # Multiclass OVR: apply threshold strategy per class
                 thresholds_ovr = []
                 for c in range(n_classes):
-                    y_binary_per_fold = [(y == self.classes_[c]).astype(int) for y in oof_y_true]
+                    y_binary_per_fold = [(y == c).astype(int) for y in oof_y_true]
                     p_c_per_fold = [p[:, c] for p in cal_probas_per_fold]
                     if self.threshold_strategy == "fold_specific":
                         tr_c = FoldSpecificThreshold.optimize(
@@ -420,14 +424,13 @@ class NestedCVClassifier(_BaseNestedCV):
                     thresholds_ovr.append(tr_c.optimal_threshold)
                 artifacts["optimal_thresholds_ovr"] = np.array(thresholds_ovr)
 
-        # --- Phase 2c: Conformal prediction ---
         if self.conformal_prediction:
             from nestkit.conformal.classifier_conformal import MondrianClassifierConformal
 
             conformal_result = MondrianClassifierConformal.fit(
                 oof_probas=cal_probas_all,
                 oof_y_true=oof_y_all,
-                classes=self.classes_,
+                classes=np.arange(self.n_classes_),
                 alpha=self.conformal_alpha,
             )
             artifacts["conformal_result"] = conformal_result
@@ -448,7 +451,7 @@ class NestedCVClassifier(_BaseNestedCV):
             effective_proba = extract_positive_proba(cal_proba)
             y_pred_default = (effective_proba >= 0.5).astype(int)
         else:
-            y_pred_default = self.classes_[np.argmax(cal_proba, axis=1)]
+            y_pred_default = np.argmax(cal_proba, axis=1)
 
         scores_default = self._compute_metrics(y_test, y_pred_default, cal_proba, is_binary)
         cm_default = confusion_matrix(y_test, y_pred_default)
@@ -457,11 +460,13 @@ class NestedCVClassifier(_BaseNestedCV):
             artifacts["calibrator"] is not None or artifacts.get("calibrators_ovr") is not None
         )
 
+        y_true_labels = self._decode(y_test)
+
         result = {
-            "y_true": y_test,
+            "y_true": y_true_labels,
             "y_proba_raw": raw_proba,
             "y_proba_calibrated": cal_proba if has_calibration else None,
-            "y_pred_default": y_pred_default,
+            "y_pred_default": self._decode(y_pred_default),
             "scores_default": scores_default,
             "confusion_matrix_default": cm_default,
             "y_pred_optimized": None,
@@ -502,8 +507,8 @@ class NestedCVClassifier(_BaseNestedCV):
                     np.argmax(above, axis=1),
                     np.argmax(cal_proba, axis=1),
                 )
-                y_pred_opt = self.classes_[idx_opt]
-            result["y_pred_optimized"] = y_pred_opt
+                y_pred_opt = idx_opt
+            result["y_pred_optimized"] = self._decode(y_pred_opt)
             result["scores_optimized"] = self._compute_metrics(
                 y_test, y_pred_opt, cal_proba, is_binary
             )
@@ -523,8 +528,8 @@ class NestedCVClassifier(_BaseNestedCV):
             result["conformal_coverage"] = float(
                 np.mean(
                     [
-                        y_test[i] in conformal_output["prediction_sets"][i]
-                        for i in range(len(y_test))
+                        y_true_labels[i] in conformal_output["prediction_sets"][i]
+                        for i in range(len(y_true_labels))
                     ]
                 )
             )

{nestkit-0.2.0 → nestkit-0.3.0}/src/nestkit/comparison/statistical_tests.py

@@ -95,17 +95,18 @@ def nadeau_bengio_corrected_ttest(
     """
     diffs = scores_a - scores_b
     n = len(diffs)
+    if n < 2:
+        raise ValueError(
+            f"The corrected t-test needs at least 2 folds to estimate a variance, got {n}."
+        )
     mean_diff = float(np.mean(diffs))
     var_diff = float(np.var(diffs, ddof=1))
 
-    # Nadeau-Bengio correction
     correction = (1.0 / n) + (n_test / n_train)
     corrected_var = correction * var_diff
     corrected_std = float(np.sqrt(corrected_var))
 
     if corrected_std == 0:
-        # When all fold differences are identical: if nonzero, evidence
-        # against H_0 is maximal; if zero, no difference exists.
         if mean_diff != 0:
             return {
                 "t_statistic": float(np.sign(mean_diff) * np.inf),
@@ -235,10 +236,15 @@ def bayesian_correlated_ttest(
     """
     diffs = scores_a - scores_b
     n = len(diffs)
+    if n < 2:
+        raise ValueError(
+            f"The Bayesian correlated t-test needs at least 2 folds to estimate a "
+            f"variance, got {n}."
+        )
     mean_diff = float(np.mean(diffs))
     std_diff = float(np.std(diffs, ddof=1))
 
-    if std_diff == 0 or n < 2:
+    if std_diff == 0:
         if abs(mean_diff) <= rope:
             return {
                 "p_a_better": 0.0,
@@ -323,7 +329,7 @@ def holm_bonferroni_correction(p_values: list[float]) -> list[float]:
     Examples
     --------
     >>> holm_bonferroni_correction([0.01, 0.04, 0.03])
-    [0.03, 0.04, 0.06]
+    [0.03, 0.06, 0.06]
     """
     n = len(p_values)
     sorted_indices = np.argsort(p_values)