classifier-toolkit 0.3.4__tar.gz → 0.3.6__tar.gz

{classifier_toolkit-0.3.4 → classifier_toolkit-0.3.6}/.github/workflows/checks.yaml

@@ -91,13 +91,27 @@ jobs:
       - name: Run linting
         run: uv run make lint
 
+  discover-test-groups:
+    runs-on: default
+    outputs:
+      test-groups: ${{ steps.list.outputs.test-groups }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: List test groups
+        id: list
+        run: |
+          groups=$(find tests -mindepth 1 -maxdepth 1 -type d ! -name '__*' ! -name '.*' -exec basename {} \; | sort | sed 's/.*/"&"/' | paste -sd, -)
+          echo "test-groups=[${groups}]" >> "$GITHUB_OUTPUT"
+          echo "Test groups: [${groups}]"
+
   test:
-    needs: dependencies
+    needs: [dependencies, discover-test-groups]
     runs-on: default
     strategy:
       fail-fast: false
       matrix:
-        test-group: [eda, feature_selection, feature_reduction, data_partition]
+        test-group: ${{ fromJSON(needs.discover-test-groups.outputs.test-groups) }}
     steps:
       - uses: actions/checkout@v6
 

{classifier_toolkit-0.3.4 → classifier_toolkit-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: classifier-toolkit
-Version: 0.3.4
+Version: 0.3.6
 Project-URL: Documentation, https://supreme-adventure-jg5qkyr.pages.github.io/
 Author-email: "senih.yilmaz" <senih.yilmaz@qonto.com>, "jeremy.fraoua" <jeremy.fraoua@qonto.com>, "gauthier.marquand" <gauthier.marquand@qonto.com>, "arnaud.alepee" <arnaud.alepee@qonto.com>
 License-File: LICENSE

classifier_toolkit-0.3.6/classifier_toolkit/calibration/__init__.py

@@ -0,0 +1,34 @@
+"""Probability calibration for multiclass (and binary) model output.
+
+Public API for the OVR histogram-binning calibrator and its reliability
+curve diagnostics — see :mod:`classifier_toolkit.calibration.ovr_calibration`
+for details.
+"""
+
+import logging
+from importlib import import_module
+from typing import Any
+
+logging.getLogger("classifier_toolkit.calibration").addHandler(logging.NullHandler())
+
+_EXPORTS: dict[str, str] = {
+    "CalibrationError": "classifier_toolkit.calibration.base",
+    "OVRHistogramCalibrator": "classifier_toolkit.calibration.ovr_calibration",
+    "plot_reliability_curves": "classifier_toolkit.calibration.reliability",
+}
+
+__all__ = list(_EXPORTS.keys())
+
+
+def __getattr__(name: str) -> Any:  # pragma: no cover
+    module_path = _EXPORTS.get(name)
+    if module_path is None:
+        raise AttributeError(
+            f"module 'classifier_toolkit.calibration' has no attribute '{name}'"
+        )
+    module = import_module(module_path)
+    return getattr(module, name)
+
+
+def __dir__():  # pragma: no cover
+    return sorted(list(globals().keys()) + __all__)

classifier_toolkit-0.3.6/classifier_toolkit/calibration/base.py

@@ -0,0 +1,5 @@
+"""Shared exception for the calibration package."""
+
+
+class CalibrationError(Exception):
+    """Base exception for calibration errors."""

classifier_toolkit-0.3.6/classifier_toolkit/calibration/ovr_calibration.py

@@ -0,0 +1,254 @@
+"""One-vs-Rest histogram-binning calibration for multiclass probabilities.
+
+Raw CatBoost/LightGBM scores are not guaranteed to be calibrated, which
+matters whenever predicted probabilities are used as probabilities (pricing,
+provisioning, or combining several models' outputs). This module calibrates
+each class independently via histogram binning, then renormalizes the
+calibrated row back to sum to 1.
+
+It calibrates the output of a *single* model across the classes it predicts.
+"""
+
+import warnings
+from typing import Dict, Optional, Sequence, Union
+
+import numpy as np
+
+from classifier_toolkit.calibration.base import CalibrationError
+
+ClassLabel = Union[str, int]
+
+
+class OVRHistogramCalibrator:
+    """Calibrate multiclass probabilities via One-vs-Rest histogram binning.
+
+    For each class independently: bin the raw predicted probability into
+    ``n_bins`` quantile-width bins, and replace it with the empirical class
+    frequency observed in that bin at fit time (histogram/binning
+    calibration — a non-parametric alternative to isotonic regression or
+    Platt scaling). Because each class is calibrated independently, the
+    calibrated row no longer sums to 1 — :meth:`transform` renormalizes it.
+
+    Sparse cells (e.g. very few observed events of a rare class in a given
+    probability bin) are shrunk toward that class's overall prevalence in
+    proportion to how little data the bin has, rather than reported as a
+    noisy raw frequency — see ``min_samples_per_bin``.
+
+    Parameters
+    ----------
+    n_bins : int, optional
+        Target number of quantile bins per class, by default 10. When many
+        scores are tied (common for a rare class, e.g. most D scores at the
+        same low value), plain quantile edges collapse; each heavily tied
+        value then gets its own bin and the remaining bins are spread over
+        the untied scores. If fewer than 3 bins can still be formed (the
+        scores take very few distinct values), a warning is raised, since
+        the calibrated output for that class is then nearly constant.
+    min_samples_per_bin : int, optional
+        Minimum training samples a bin must contain before its empirical
+        frequency is trusted as-is; bins with fewer samples are shrunk
+        toward the class's overall prevalence in proportion to
+        ``n_in_bin / min_samples_per_bin``. By default 20.
+
+    Attributes
+    ----------
+    classes_ : Optional[np.ndarray]
+        Class labels, in the order matching probability columns. Set by fit.
+    bin_edges_ : Dict[ClassLabel, np.ndarray]
+        Per-class quantile bin edges from fit (outer edges are +/-inf so
+        every future probability falls inside some bin).
+    bin_values_ : Dict[ClassLabel, np.ndarray]
+        Per-class calibrated (possibly shrunk) frequency for each bin.
+    class_prior_ : Dict[ClassLabel, float]
+        Per-class overall prevalence at fit time, used for shrinkage.
+    """
+
+    def __init__(self, n_bins: int = 10, min_samples_per_bin: int = 20) -> None:
+        self.n_bins = n_bins
+        self.min_samples_per_bin = min_samples_per_bin
+        self.classes_: Optional[np.ndarray] = None
+        self.bin_edges_: Dict[ClassLabel, np.ndarray] = {}
+        self.bin_values_: Dict[ClassLabel, np.ndarray] = {}
+        self.class_prior_: Dict[ClassLabel, float] = {}
+
+    def _quantile_edges(self, p: np.ndarray) -> np.ndarray:
+        quantiles = np.linspace(0.0, 1.0, self.n_bins + 1)
+        edges = np.unique(np.quantile(p, quantiles))
+        if len(edges) - 1 < self.n_bins:
+            edges = self._tie_aware_edges(p)
+        if len(edges) < 2:
+            edges = np.array([0.0, 1.0])
+        edges = edges.astype(float)
+        edges[0] = -np.inf
+        edges[-1] = np.inf
+        return edges
+
+    def _tie_aware_edges(self, p: np.ndarray) -> np.ndarray:
+        """Quantile edges that give each heavily tied score its own bin.
+
+        A value is "heavily tied" when it holds more than ``1 / n_bins`` of
+        the samples, i.e. more than one quantile bin's worth. Such a value
+        collapses neighbouring quantile edges onto itself, so it is isolated
+        in its own bin (``[v, next float after v)``) and the rest of the bin
+        budget is spent on quantiles of the other scores.
+        """
+        values, counts = np.unique(p, return_counts=True)
+        spikes = values[counts > len(p) / self.n_bins]
+        rest = p[~np.isin(p, spikes)]
+        n_rest_bins = max(self.n_bins - len(spikes), 1)
+        edges = list(spikes) + [np.nextafter(v, np.inf) for v in spikes]
+        if rest.size:
+            edges += list(np.quantile(rest, np.linspace(0.0, 1.0, n_rest_bins + 1)))
+        return np.unique(np.asarray(edges, dtype=float))
+
+    @staticmethod
+    def _bin_index(p: np.ndarray, edges: np.ndarray) -> np.ndarray:
+        return np.clip(np.digitize(p, edges[1:-1], right=False), 0, len(edges) - 2)
+
+    def fit(
+        self,
+        y_true: Union[Sequence, np.ndarray],
+        y_proba: np.ndarray,
+        classes: Optional[Sequence[ClassLabel]] = None,
+    ) -> "OVRHistogramCalibrator":
+        """Fit per-class histogram bins from raw predictions.
+
+        Parameters
+        ----------
+        y_true : array-like of shape (n_samples,)
+            True class labels.
+        y_proba : np.ndarray of shape (n_samples, n_classes)
+            Raw (uncalibrated) predicted probabilities, e.g. from
+            ``model.predict_proba(X)``.
+        classes : Sequence[ClassLabel], optional
+            Class labels in the order matching ``y_proba``'s columns
+            (typically ``model.classes_``). If None, inferred as the sorted
+            unique values of ``y_true`` — only safe if every class appears
+            in ``y_true`` and ``y_proba``'s columns are already in that
+            sorted order.
+
+        Returns
+        -------
+        OVRHistogramCalibrator
+            The fitted calibrator.
+        """
+        y_true = np.asarray(y_true)
+        y_proba = self._check_proba(y_proba)
+        if classes is None:
+            classes = np.unique(y_true)
+        classes = np.asarray(classes)
+        if y_proba.shape[1] != len(classes):
+            raise CalibrationError(
+                f"y_proba has {y_proba.shape[1]} columns but {len(classes)} "
+                "classes were given."
+            )
+        if y_proba.shape[0] != len(y_true):
+            raise CalibrationError(
+                f"y_proba has {y_proba.shape[0]} rows but y_true has {len(y_true)}."
+            )
+
+        self.classes_ = classes
+        self.bin_edges_ = {}
+        self.bin_values_ = {}
+        self.class_prior_ = {}
+
+        for i, c in enumerate(classes):
+            y_bin = (y_true == c).astype(int)
+            p = y_proba[:, i]
+            prior = float(y_bin.mean()) if len(y_bin) > 0 else 0.0
+            self.class_prior_[c] = prior
+
+            edges = self._quantile_edges(p)
+            bin_idx = self._bin_index(p, edges)
+            n_populated = len(np.unique(bin_idx))
+            if n_populated < min(self.n_bins, 3):
+                warnings.warn(
+                    f"Class {c!r}: only {n_populated} calibration bin(s) could "
+                    f"be formed (n_bins={self.n_bins}) because its scores take "
+                    "very few distinct values; its calibrated probability will "
+                    "be nearly constant.",
+                    stacklevel=2,
+                )
+
+            values = np.empty(len(edges) - 1, dtype=float)
+            for b in range(len(edges) - 1):
+                mask = bin_idx == b
+                n_in_bin = int(mask.sum())
+                if n_in_bin == 0:
+                    values[b] = prior
+                    continue
+                observed = float(y_bin[mask].mean())
+                if n_in_bin < self.min_samples_per_bin:
+                    weight = n_in_bin / self.min_samples_per_bin
+                    values[b] = weight * observed + (1 - weight) * prior
+                else:
+                    values[b] = observed
+            self.bin_edges_[c] = edges
+            self.bin_values_[c] = values
+
+        return self
+
+    def transform(self, y_proba: np.ndarray) -> np.ndarray:
+        """Calibrate raw probabilities and renormalize to sum to 1.
+
+        Parameters
+        ----------
+        y_proba : np.ndarray of shape (n_samples, n_classes)
+            Raw predicted probabilities, columns matching ``classes_``.
+
+        Returns
+        -------
+        np.ndarray of shape (n_samples, n_classes)
+            Calibrated probabilities; each row sums to 1. A row whose
+            calibrated values are all exactly 0 (every relevant bin observed
+            no events) carries no information, so it falls back to the
+            class priors from fit.
+        """
+        if self.classes_ is None:
+            raise CalibrationError("Calibrator is not fitted. Call fit() first.")
+        y_proba = self._check_proba(y_proba)
+        if y_proba.shape[1] != len(self.classes_):
+            raise CalibrationError(
+                f"y_proba has {y_proba.shape[1]} columns but the calibrator "
+                f"was fit on {len(self.classes_)} classes."
+            )
+
+        calibrated = np.empty_like(y_proba)
+        for i, c in enumerate(self.classes_):
+            edges = self.bin_edges_[c]
+            values = self.bin_values_[c]
+            bin_idx = self._bin_index(y_proba[:, i], edges)
+            calibrated[:, i] = values[bin_idx]
+
+        row_sums = calibrated.sum(axis=1, keepdims=True)
+        empty_rows = row_sums[:, 0] == 0
+        if empty_rows.any():
+            priors = np.array([self.class_prior_[c] for c in self.classes_])
+            if priors.sum() == 0:
+                priors = np.ones(len(self.classes_))
+            calibrated[empty_rows] = priors
+            row_sums = calibrated.sum(axis=1, keepdims=True)
+        return calibrated / row_sums
+
+    @staticmethod
+    def _check_proba(y_proba: np.ndarray) -> np.ndarray:
+        y_proba = np.asarray(y_proba, dtype=float)
+        if y_proba.ndim != 2:
+            raise CalibrationError(
+                f"y_proba must be 2D (n_samples, n_classes); got shape {y_proba.shape}."
+            )
+        if np.isnan(y_proba).any():
+            raise CalibrationError(
+                f"y_proba contains {int(np.isnan(y_proba).sum())} NaN value(s); "
+                "calibration needs a probability for every sample and class."
+            )
+        return y_proba
+
+    def fit_transform(
+        self,
+        y_true: Union[Sequence, np.ndarray],
+        y_proba: np.ndarray,
+        classes: Optional[Sequence[ClassLabel]] = None,
+    ) -> np.ndarray:
+        """Fit the calibrator and calibrate the same data in one step."""
+        return self.fit(y_true, y_proba, classes=classes).transform(y_proba)

classifier_toolkit-0.3.6/classifier_toolkit/calibration/reliability.py

@@ -0,0 +1,109 @@
+"""Reliability curve diagnostics for OVR calibration.
+
+One reliability (calibration) curve per class, each an
+independent binary "class *c* vs rest" diagnostic — matching the OVR
+calibration approach itself. Plots raw and (optionally) calibrated curves
+together so the effect of calibration is visible directly.
+"""
+
+import math
+from typing import Optional, Sequence, Union
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.calibration import calibration_curve
+
+ClassLabel = Union[str, int]
+
+
+def plot_reliability_curves(
+    y_true: Union[Sequence, np.ndarray],
+    y_proba_raw: np.ndarray,
+    y_proba_calibrated: Optional[np.ndarray] = None,
+    classes: Optional[Sequence[ClassLabel]] = None,
+    n_bins: int = 10,
+    pos_label: Optional[ClassLabel] = None,
+    n_cols: int = 3,
+    show: bool = True,
+) -> plt.Figure:
+    """Plot a reliability curve per class, one subplot each.
+
+    Parameters
+    ----------
+    y_true : array-like of shape (n_samples,)
+        True class labels.
+    y_proba_raw : np.ndarray of shape (n_samples, n_classes)
+        Raw (uncalibrated) predicted probabilities.
+    y_proba_calibrated : np.ndarray of shape (n_samples, n_classes), optional
+        Calibrated predicted probabilities (e.g. from
+        :meth:`OVRHistogramCalibrator.transform`), plotted alongside the raw
+        curve for comparison. If None, only the raw curve is shown.
+    classes : Sequence[ClassLabel], optional
+        Class labels in the order matching the probability columns. If
+        None, inferred as the sorted unique values of ``y_true``.
+    n_bins : int, optional
+        Number of quantile bins per curve, by default 10.
+    pos_label : ClassLabel, optional
+        A class of interest (e.g. the default class). If given and present
+        in ``classes``, that subplot's title is bolded in red to keep it
+        visually prominent, consistent with the rest of the multiclass
+        toolkit. By default None.
+    n_cols : int, optional
+        Subplot grid columns, by default 3.
+    show : bool, optional
+        Whether to call ``plt.show()``, by default True.
+
+    Returns
+    -------
+    matplotlib.figure.Figure
+    """
+    y_true = np.asarray(y_true)
+    y_proba_raw = np.asarray(y_proba_raw)
+    if classes is None:
+        classes = np.unique(y_true)
+    classes = list(classes)
+    n_classes = len(classes)
+
+    n_cols = min(n_cols, n_classes)
+    n_rows = math.ceil(n_classes / n_cols)
+    fig, axes = plt.subplots(
+        n_rows, n_cols, figsize=(5 * n_cols, 4 * n_rows), squeeze=False
+    )
+    axes_flat = axes.flatten()
+
+    for i, c in enumerate(classes):
+        ax = axes_flat[i]
+        y_bin = (y_true == c).astype(int)
+
+        frac_pos, mean_pred = calibration_curve(
+            y_bin, y_proba_raw[:, i], n_bins=n_bins, strategy="quantile"
+        )
+        ax.plot(mean_pred, frac_pos, marker="o", label="Raw")
+
+        if y_proba_calibrated is not None:
+            frac_pos_cal, mean_pred_cal = calibration_curve(
+                y_bin,
+                np.asarray(y_proba_calibrated)[:, i],
+                n_bins=n_bins,
+                strategy="quantile",
+            )
+            ax.plot(mean_pred_cal, frac_pos_cal, marker="s", label="Calibrated")
+
+        ax.plot([0, 1], [0, 1], linestyle="--", color="gray", label="Perfect")
+        ax.set_xlabel("Mean predicted probability")
+        ax.set_ylabel("Observed frequency")
+        title = f"Class {c}"
+        if pos_label is not None and c == pos_label:
+            ax.set_title(title, fontweight="bold", color="red")
+        else:
+            ax.set_title(title)
+        ax.legend(fontsize=8)
+
+    for i in range(n_classes, n_rows * n_cols):
+        axes_flat[i].set_visible(False)
+
+    fig.suptitle("Reliability Curves (One-vs-Rest per class)", fontsize=14, y=1.02)
+    plt.tight_layout()
+    if show:
+        plt.show()
+    return fig

{classifier_toolkit-0.3.4 → classifier_toolkit-0.3.6}/classifier_toolkit/eda/eda_toolkit.py

@@ -49,6 +49,7 @@ class EDAToolkit:
         id_column: Optional[str] = None,
         to_be_enc: Optional[List[str]] = None,
         warning_system: Optional[WarningSystem] = None,
+        pos_label: Optional[Union[str, int]] = None,
     ) -> None:
         """
         Initialize the EDAToolkit class with data and configuration settings.
@@ -73,6 +74,10 @@ class EDAToolkit:
             List of columns to be encoded, by default None.
         warning_system : Optional[WarningSystem], optional
             The warning system to use, by default None.
+        pos_label : Union[str, int], optional
+            The target class of interest (e.g. the default class). Forwarded
+            to :class:`Visualizations` to highlight/select that class in the
+            per-class EDA views. By default None.
         """
         self.dataframe = dataframe
         self.target_column = target_column or ""
@@ -114,6 +119,7 @@ class EDAToolkit:
             self.target_column,
             self.numerical_columns,
             self.categorical_columns,
+            pos_label=pos_label,
         )
         self.univariate_analysis = UnivariateAnalysis(
             self.dataframe, self.target_column, self.numerical_columns