classifier-toolkit 0.3.3__tar.gz → 0.3.5__tar.gz

{classifier_toolkit-0.3.3 → classifier_toolkit-0.3.5}/.gitignore

@@ -78,6 +78,7 @@ target/
 # Jupyter Notebook
 .ipynb_checkpoints
 *.ipynb
+!examples/*.ipynb
 
 # IPython
 profile_default/

{classifier_toolkit-0.3.3 → classifier_toolkit-0.3.5}/PKG-INFO

@@ -1,7 +1,7 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: classifier-toolkit
-Version: 0.3.3
-Project-URL: Documentation, http://classifier-toolkit.gh-pages.qonto.co/
+Version: 0.3.5
+Project-URL: Documentation, https://supreme-adventure-jg5qkyr.pages.github.io/
 Author-email: "senih.yilmaz" <senih.yilmaz@qonto.com>, "jeremy.fraoua" <jeremy.fraoua@qonto.com>, "gauthier.marquand" <gauthier.marquand@qonto.com>, "arnaud.alepee" <arnaud.alepee@qonto.com>
 License-File: LICENSE
 Requires-Python: <3.14,>=3.9

{classifier_toolkit-0.3.3 → classifier_toolkit-0.3.5}/classifier_toolkit/feature_reduction/predictive_power.py

@@ -13,6 +13,13 @@ A feature is removed when *either* metric falls below its threshold.
 Features with ``NaN`` metrics (constant, insufficient data, computation error)
 are also removed — they cannot be trusted.
 
+For **multiclass targets** (``k > 2``) the metrics are computed One-vs-Rest
+(OVR) per destination class and the **maximum** across classes is used as the
+aggregate score. A feature is kept if it clears the threshold for *at least
+one* class — this ensures rare-class predictors (e.g. S2→D) are not discarded
+because they are uninformative for the common transitions. Both binary and
+multiclass paths reduce to the same result when ``k == 2``.
+
 Two classes are exposed:
 
 - :class:`PredictivePowerAnalyser`: computes per-feature Gini and PR-AUC lift
@@ -25,7 +32,7 @@ Two classes are exposed:
 
 import logging
 import warnings
-from typing import List, Optional, Tuple
+from typing import Any, Optional
 
 import numpy as np
 import pandas as pd
@@ -40,8 +47,8 @@ warnings.filterwarnings("ignore", category=UserWarning, module="sklearn")
 _PREVIEW_LIMIT = 5
 
 
-def _gini(feature: pd.Series, y: pd.Series) -> Tuple[float, str]:
-    """Direction-agnostic Gini for one numerical feature.
+def _gini(feature: pd.Series, y: pd.Series) -> tuple[float, str]:
+    """Direction-agnostic Gini for one numerical feature against a binary target.
 
     Returns (gini, direction). Direction is ``"positive"``, ``"negative"``,
     ``"n/a"`` (constant / insufficient data), or ``"error"``.
@@ -59,8 +66,35 @@ def _gini(feature: pd.Series, y: pd.Series) -> Tuple[float, str]:
         return np.nan, "error"
 
 
-def _pr_auc_lift(feature: pd.Series, y: pd.Series) -> Tuple[float, float, str]:
-    """PR-AUC and lift (%) for one numerical feature.
+def _gini_ovr(feature: pd.Series, y: pd.Series) -> tuple[float, str, dict[Any, float]]:
+    """OVR Gini for one feature against a multiclass target.
+
+    Collapses to a c-vs-rest binary problem for each class ``c`` and delegates
+    to :func:`_gini`. The aggregate score is the maximum Gini across classes so
+    that a feature predictive for only one rare class is not discarded.
+
+    Returns ``(max_gini, best_direction, {class: gini_c})``.
+    """
+    classes = sorted(y.dropna().unique())
+    per_class: dict[Any, float] = {}
+    best_gini = -np.inf
+    best_direction = "n/a"
+
+    for c in classes:
+        y_binary = (y == c).astype(int)
+        g, direction = _gini(feature, y_binary)
+        per_class[c] = g
+        if not np.isnan(g) and g > best_gini:
+            best_gini = g
+            best_direction = direction
+
+    if best_gini == -np.inf:
+        return np.nan, "n/a", per_class
+    return float(best_gini), best_direction, per_class
+
+
+def _pr_auc_lift(feature: pd.Series, y: pd.Series) -> tuple[float, float, str]:
+    """PR-AUC and lift (%) for one numerical feature against a binary target.
 
     Lift = (PR-AUC - baseline) / baseline x 100, where baseline = prevalence.
     Both directions are tried; the better one is kept.
@@ -85,6 +119,37 @@ def _pr_auc_lift(feature: pd.Series, y: pd.Series) -> Tuple[float, float, str]:
         return np.nan, np.nan, "error"
 
 
+def _pr_auc_lift_ovr(
+    feature: pd.Series, y: pd.Series
+) -> tuple[float, float, str, dict[Any, float]]:
+    """OVR PR-AUC lift for one feature against a multiclass target.
+
+    Collapses to a c-vs-rest binary problem for each class ``c`` and delegates
+    to :func:`_pr_auc_lift`. The aggregate score is the maximum PR-AUC across
+    classes.
+
+    Returns ``(max_prauc, best_lift_pct, best_direction, {class: prauc_c})``.
+    """
+    classes = sorted(y.dropna().unique())
+    per_class: dict[Any, float] = {}
+    best_prauc = -np.inf
+    best_lift: float = np.nan
+    best_direction = "n/a"
+
+    for c in classes:
+        y_binary = (y == c).astype(int)
+        pr_auc, lift_pct, direction = _pr_auc_lift(feature, y_binary)
+        per_class[c] = pr_auc
+        if not np.isnan(pr_auc) and pr_auc > best_prauc:
+            best_prauc = pr_auc
+            best_lift = lift_pct
+            best_direction = direction
+
+    if best_prauc == -np.inf:
+        return np.nan, np.nan, "n/a", per_class
+    return float(best_prauc), float(best_lift), best_direction, per_class
+
+
 # ---------------------------------------------------------------------------
 # Standalone public helpers
 # ---------------------------------------------------------------------------
@@ -111,16 +176,21 @@ _PRAUC_LIFT_LABELS = [
 def calculate_feature_gini(
     df: pd.DataFrame,
     target: str,
-    exclude: Optional[List[str]] = None,
+    exclude: Optional[list[str]] = None,
 ) -> pd.DataFrame:
     """Compute direction-agnostic Gini for every numeric feature.
 
+    For binary targets the Gini is computed directly. For multiclass targets
+    (``k > 2``) One-vs-Rest Gini is computed per class and the maximum across
+    classes is reported as ``Gini``; individual per-class values appear in
+    additional ``Gini_{class}`` columns.
+
     Parameters
     ----------
     df : pd.DataFrame
         DataFrame containing features and target.
     target : str
-        Name of the binary target column.
+        Name of the target column (binary 0/1 or multiclass integer/string).
     exclude : List[str], optional
         Additional columns to exclude from scoring.
 
@@ -128,30 +198,37 @@ def calculate_feature_gini(
     -------
     pd.DataFrame
         One row per numeric feature, sorted by ``Gini`` descending, with
-        columns ``Feature``, ``Gini``, ``Direction``, ``Interpretation``.
+        columns ``Feature``, ``Gini``, ``Direction``, ``Interpretation`` and,
+        for multiclass targets, additional ``Gini_{class}`` columns.
     """
     skip = set(exclude or []) | {target}
+    k = df[target].nunique()
     rows = []
     for col in df.columns:
         if col in skip:
             continue
         if not np.issubdtype(df[col].dtype, np.number):
             continue
-        gini, direction = _gini(df[col], df[target])
+        if k > 2:
+            gini, direction, per_class = _gini_ovr(df[col], df[target])
+        else:
+            gini, direction = _gini(df[col], df[target])
+            per_class = {}
         if np.isnan(gini):
             interp = "Cannot be calculated"
         else:
             interp = next(
                 label for threshold, label in _GINI_LABELS if gini < threshold
             )
-        rows.append(
-            {
-                "Feature": col,
-                "Gini": gini,
-                "Direction": direction,
-                "Interpretation": interp,
-            }
-        )
+        row: dict[str, Any] = {
+            "Feature": col,
+            "Gini": gini,
+            "Direction": direction,
+            "Interpretation": interp,
+        }
+        for c, g in per_class.items():
+            row[f"Gini_{c}"] = g
+        rows.append(row)
     return (
         pd.DataFrame(rows)
         .sort_values("Gini", ascending=False, na_position="last")
@@ -162,16 +239,22 @@ def calculate_feature_gini(
 def calculate_feature_prauc(
     df: pd.DataFrame,
     target: str,
-    exclude: Optional[List[str]] = None,
+    exclude: Optional[list[str]] = None,
 ) -> pd.DataFrame:
     """Compute PR-AUC and lift over baseline for every numeric feature.
 
+    For binary targets the PR-AUC is computed directly. For multiclass targets
+    (``k > 2``) One-vs-Rest PR-AUC is computed per class and the maximum across
+    classes is reported as ``PR_AUC``; individual per-class values appear in
+    additional ``PR_AUC_{class}`` columns. ``Baseline_PR_AUC`` reports the
+    macro-average class prevalence in the multiclass case.
+
     Parameters
     ----------
     df : pd.DataFrame
         DataFrame containing features and target.
     target : str
-        Name of the binary target column.
+        Name of the target column (binary 0/1 or multiclass integer/string).
     exclude : List[str], optional
         Additional columns to exclude from scoring.
 
@@ -180,17 +263,25 @@ def calculate_feature_prauc(
     pd.DataFrame
         One row per numeric feature, sorted by ``PR_AUC`` descending, with
         columns ``Feature``, ``PR_AUC``, ``Baseline_PR_AUC``, ``Lift_%``,
-        ``Direction``, ``Interpretation``.
+        ``Direction``, ``Interpretation`` and, for multiclass targets,
+        additional ``PR_AUC_{class}`` columns.
     """
     skip = set(exclude or []) | {target}
-    baseline = float(df[target].mean())
+    k = df[target].nunique()
+    baseline = float(df[target].mean()) if k == 2 else float(1 / k)
     rows = []
     for col in df.columns:
         if col in skip:
             continue
         if not np.issubdtype(df[col].dtype, np.number):
             continue
-        pr_auc, lift_pct, direction = _pr_auc_lift(df[col], df[target])
+        if k > 2:
+            pr_auc, lift_pct, direction, per_class = _pr_auc_lift_ovr(
+                df[col], df[target]
+            )
+        else:
+            pr_auc, lift_pct, direction = _pr_auc_lift(df[col], df[target])
+            per_class = {}
         if np.isnan(pr_auc):
             interp = "Cannot be calculated"
         else:
@@ -198,16 +289,17 @@ def calculate_feature_prauc(
             interp = next(
                 label for threshold, label in _PRAUC_LIFT_LABELS if ratio < threshold
             )
-        rows.append(
-            {
-                "Feature": col,
-                "PR_AUC": pr_auc,
-                "Baseline_PR_AUC": baseline,
-                "Lift_%": lift_pct,
-                "Direction": direction,
-                "Interpretation": interp,
-            }
-        )
+        row: dict[str, Any] = {
+            "Feature": col,
+            "PR_AUC": pr_auc,
+            "Baseline_PR_AUC": baseline,
+            "Lift_%": lift_pct,
+            "Direction": direction,
+            "Interpretation": interp,
+        }
+        for c, p in per_class.items():
+            row[f"PR_AUC_{c}"] = p
+        rows.append(row)
     return (
         pd.DataFrame(rows)
         .sort_values("PR_AUC", ascending=False, na_position="last")
@@ -218,7 +310,7 @@ def calculate_feature_prauc(
 def calculate_feature_predictive_metrics(
     df: pd.DataFrame,
     target: str,
-    exclude: Optional[List[str]] = None,
+    exclude: Optional[list[str]] = None,
 ) -> pd.DataFrame:
     """Compute Gini and PR-AUC metrics for every numeric feature.
 
@@ -273,7 +365,8 @@ class PredictivePowerAnalyser:
         :meth:`transform`.
     power_stats_ : Optional[pd.DataFrame]
         Per-feature statistics with columns ``feature``, ``gini``,
-        ``pr_auc``, ``pr_auc_lift_pct``, ``direction``, ``removed``.
+        ``pr_auc``, ``pr_auc_lift_pct``, ``direction``, ``removed`` and,
+        for multiclass targets, additional ``gini_{class}`` columns.
         ``None`` if fit was called with no features.
     """
 
@@ -285,9 +378,9 @@ class PredictivePowerAnalyser:
         self.gini_threshold = gini_threshold
         self.prauc_lift_threshold = prauc_lift_threshold
 
-        self.removed_features_: List[str] = []
-        self.remaining_features_: List[str] = []
-        self.skipped_features_: List[str] = []
+        self.removed_features_: list[str] = []
+        self.remaining_features_: list[str] = []
+        self.skipped_features_: list[str] = []
         self.power_stats_: Optional[pd.DataFrame] = None
         self._fitted: bool = False
 
@@ -295,17 +388,24 @@ class PredictivePowerAnalyser:
         self,
         X: pd.DataFrame,
         y: pd.Series,
-        numerical_features: Optional[List[str]] = None,
+        numerical_features: Optional[list[str]] = None,
     ) -> "PredictivePowerAnalyser":
         """
         Compute Gini and PR-AUC lift for each feature and flag weak ones.
 
+        For binary targets (``k == 2``) the metrics are computed directly. For
+        multiclass targets (``k > 2``) One-vs-Rest metrics are computed per
+        class and the maximum across classes is used as the aggregate score —
+        a feature is kept if it clears the threshold for *at least one* class.
+        Per-class Gini values are stored as ``gini_{class}`` columns in
+        ``power_stats_`` for downstream use (e.g. Stage 1.5 direction logic).
+
         Parameters
         ----------
         X : pd.DataFrame
             Feature matrix.
         y : pd.Series
-            Binary target variable (0/1).
+            Target variable (binary 0/1 or multiclass integer/string).
         numerical_features : List[str], optional
             Numerical columns to evaluate. If ``None``, numeric columns are
             auto-detected from dtypes; non-numeric columns are added to
@@ -341,11 +441,17 @@ class PredictivePowerAnalyser:
             self._fitted = True
             return self
 
+        k = y.nunique()
         rows = []
 
         for col in cols_to_score:
-            gini, gini_dir = _gini(X[col], y)
-            pr_auc, lift_pct, pr_dir = _pr_auc_lift(X[col], y)
+            if k > 2:
+                gini, gini_dir, per_class_gini = _gini_ovr(X[col], y)
+                pr_auc, lift_pct, pr_dir, _ = _pr_auc_lift_ovr(X[col], y)
+            else:
+                gini, gini_dir = _gini(X[col], y)
+                pr_auc, lift_pct, pr_dir = _pr_auc_lift(X[col], y)
+                per_class_gini = {}
 
             direction = gini_dir if gini_dir not in ("n/a", "error") else pr_dir
 
@@ -359,16 +465,17 @@ class PredictivePowerAnalyser:
             if should_remove:
                 self.removed_features_.append(col)
 
-            rows.append(
-                {
-                    "feature": col,
-                    "gini": gini,
-                    "pr_auc": pr_auc,
-                    "pr_auc_lift_pct": lift_pct,
-                    "direction": direction,
-                    "removed": should_remove,
-                }
-            )
+            row: dict[str, Any] = {
+                "feature": col,
+                "gini": gini,
+                "pr_auc": pr_auc,
+                "pr_auc_lift_pct": lift_pct,
+                "direction": direction,
+                "removed": should_remove,
+            }
+            for c, g in per_class_gini.items():
+                row[f"gini_{c}"] = g
+            rows.append(row)
 
         self.remaining_features_ = [
             c for c in cols_to_score if c not in self.removed_features_
@@ -413,7 +520,7 @@ class PredictivePowerAnalyser:
         self,
         X: pd.DataFrame,
         y: pd.Series,
-        numerical_features: Optional[List[str]] = None,
+        numerical_features: Optional[list[str]] = None,
     ) -> pd.DataFrame:
         """
         Fit the analyser and transform the dataset in one step.
@@ -423,7 +530,7 @@ class PredictivePowerAnalyser:
         X : pd.DataFrame
             Feature matrix.
         y : pd.Series
-            Binary target variable.
+            Target variable (binary 0/1 or multiclass integer/string).
         numerical_features : List[str], optional
             Columns to evaluate.
 
@@ -519,7 +626,7 @@ class WeakPredictiveFilter(BaseFeatureReducer):
 
     def __init__(
         self,
-        numerical_features: Optional[List[str]] = None,
+        numerical_features: Optional[list[str]] = None,
         gini_threshold: float = 0.01,
         prauc_lift_threshold: float = 0.0,
     ) -> None:
@@ -543,7 +650,7 @@ class WeakPredictiveFilter(BaseFeatureReducer):
         X : pd.DataFrame
             Feature matrix.
         y : pd.Series
-            Binary target variable (0/1). Required.
+            Target variable (binary 0/1 or multiclass integer/string). Required.
         **kwargs
             Ignored.
 

classifier_toolkit-0.3.5/classifier_toolkit/feature_selection/utils/scoring.py

@@ -0,0 +1,226 @@
+from typing import Callable, Dict, Union
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+from sklearn.metrics import (
+    accuracy_score,
+    average_precision_score,
+    confusion_matrix,
+    f1_score,
+    make_scorer,
+    precision_score,
+    recall_score,
+    roc_auc_score,
+)
+from sklearn.preprocessing import label_binarize
+
+
+def roc_auc_ovr(y_true: ArrayLike, y_pred_proba: ArrayLike) -> float:
+    """
+    ROC-AUC that transparently handles both binary and multiclass targets.
+
+    Binary targets (``k <= 2``) use today's exact behaviour (probability of
+    the positive class). Multiclass targets (``k > 2``) use One-vs-Rest with
+    macro averaging, which reduces to the binary formula when ``k == 2``.
+
+    Parameters
+    ----------
+    y_true : ArrayLike
+        True labels.
+    y_pred_proba : ArrayLike
+        Predicted class probabilities from ``predict_proba``.
+
+    Returns
+    -------
+    float
+        ROC-AUC score.
+    """
+    y_pred_proba = np.asarray(y_pred_proba)
+    if y_pred_proba.ndim == 1 or y_pred_proba.shape[1] <= 2:
+        proba = y_pred_proba[:, 1] if y_pred_proba.ndim > 1 else y_pred_proba
+        return float(roc_auc_score(y_true, proba))
+    return float(
+        roc_auc_score(y_true, y_pred_proba, multi_class="ovr", average="macro")
+    )
+
+
+def macro_average_precision_score(y_true: ArrayLike, y_pred_proba: ArrayLike) -> float:
+    """
+    Macro-averaged One-vs-Rest PR-AUC across all classes.
+
+    Binary targets (``k <= 2``) use today's exact ``average_precision_score``
+    behaviour. Multiclass targets (``k > 2``) binarize each class against the
+    rest and average the per-class PR-AUC.
+
+    Parameters
+    ----------
+    y_true : ArrayLike
+        True labels.
+    y_pred_proba : ArrayLike
+        Predicted class probabilities from ``predict_proba``.
+
+    Returns
+    -------
+    float
+        Macro-averaged PR-AUC score.
+    """
+    y_pred_proba = np.asarray(y_pred_proba)
+    if y_pred_proba.ndim == 1 or y_pred_proba.shape[1] <= 2:
+        proba = y_pred_proba[:, 1] if y_pred_proba.ndim > 1 else y_pred_proba
+        return float(average_precision_score(y_true, proba))
+
+    classes = np.unique(y_true)
+    y_true_bin = label_binarize(y_true, classes=classes)
+    per_class_scores = [
+        average_precision_score(y_true_bin[:, i], y_pred_proba[:, i])
+        for i in range(len(classes))
+    ]
+    return float(np.mean(per_class_scores))
+
+
+def make_d_class_pr_auc_scorer(d_class_label: Union[str, int]) -> Callable:
+    """
+    Build a PR-AUC scorer for a single class treated as One-vs-Rest positive.
+
+    Used to track the "D" (default) class separately from the primary
+    macro-averaged metric, since it is the class that ultimately gets priced.
+
+    Parameters
+    ----------
+    d_class_label : Union[str, int]
+        The class value that represents the "D" (default) state.
+
+    Returns
+    -------
+    Callable
+        A function ``(y_true, y_pred_proba, classes) -> float`` computing the
+        OVR PR-AUC for ``d_class_label``.
+    """
+
+    def _d_class_pr_auc(
+        y_true: ArrayLike, y_pred_proba: ArrayLike, classes: ArrayLike
+    ) -> float:
+        classes = np.asarray(classes)
+        y_pred_proba = np.asarray(y_pred_proba)
+        matches = np.where(classes == d_class_label)[0]
+        if len(matches) == 0:
+            raise ValueError(
+                f"d_class_label={d_class_label!r} not found among classes {classes.tolist()}"
+            )
+        class_idx = matches[0]
+        y_true_bin = (np.asarray(y_true) == d_class_label).astype(int)
+        return float(average_precision_score(y_true_bin, y_pred_proba[:, class_idx]))
+
+    return _d_class_pr_auc
+
+
+# Define scorers with updated best practices
+SCORERS = {
+    "accuracy": make_scorer(accuracy_score),
+    "f1": make_scorer(f1_score, average="weighted"),
+    "precision": make_scorer(precision_score, average="weighted"),
+    "recall": make_scorer(recall_score, average="weighted"),
+    "roc_auc": make_scorer(roc_auc_ovr, response_method="predict_proba"),
+    "average_precision": make_scorer(
+        average_precision_score, response_method="predict_proba"
+    ),
+    "macro_average_precision": make_scorer(
+        macro_average_precision_score, response_method="predict_proba"
+    ),
+}
+
+# Keep the original scoring functions for backward compatibility
+SCORING_FUNCTIONS: Dict[str, Callable] = {
+    "accuracy": accuracy_score,
+    "f1": f1_score,
+    "precision": precision_score,
+    "recall": recall_score,
+    "roc_auc": roc_auc_score,
+    "average_precision": average_precision_score,
+    "macro_average_precision": macro_average_precision_score,
+}
+
+
+def get_scorer(metric: Union[str, Callable]) -> Callable:
+    """
+    Get a scorer function for the specified metric.
+
+    Parameters
+    ----------
+    metric : Union[str, Callable]
+        The metric to use for scoring. Can be a string identifier or a callable.
+
+    Returns
+    -------
+    Callable
+        A scorer function compatible with scikit-learn.
+
+    Raises
+    ------
+    ValueError
+        If the metric is not recognized.
+    """
+    if isinstance(metric, str):
+        if metric in SCORERS:
+            return SCORERS[metric]
+        elif metric in SCORING_FUNCTIONS:
+            # For backward compatibility
+            return make_scorer(SCORING_FUNCTIONS[metric])
+        else:
+            raise ValueError(
+                f"Unrecognized metric: {metric}. Available metrics are: {', '.join(SCORERS.keys())}"
+            )
+    elif callable(metric):
+        return make_scorer(metric)
+    else:
+        raise ValueError("metric must be either a string or a callable")
+
+
+def false_positive_rate(y_true: ArrayLike, y_pred: ArrayLike) -> float:
+    """
+    Calculate the false positive rate.
+
+    Parameters
+    ----------
+    y_true : ArrayLike
+        True labels.
+    y_pred : ArrayLike
+        Predicted labels.
+
+    Returns
+    -------
+    float
+        False positive rate.
+    """
+    tn, fp, _, _ = confusion_matrix(y_true, y_pred).ravel()
+    return fp / (fp + tn)
+
+
+def true_positive_rate(
+    y_true: ArrayLike, y_pred: ArrayLike
+) -> Union[float, NDArray[np.float64]]:
+    """
+    Calculate the true positive rate (recall/sensitivity).
+
+    Parameters
+    ----------
+    y_true : ArrayLike
+        True labels.
+    y_pred : ArrayLike
+        Predicted labels.
+
+    Returns
+    -------
+    Union[float, NDArray[np.float64]]
+        True positive rate.
+    """
+    return recall_score(y_true, y_pred)  # type: ignore
+
+
+# Add custom scorers for FPR and TPR
+SCORERS["fpr"] = make_scorer(false_positive_rate, greater_is_better=False)
+SCORERS["tpr"] = make_scorer(true_positive_rate, greater_is_better=True)
+
+# Add to SCORING_FUNCTIONS for backward compatibility
+SCORING_FUNCTIONS["fpr"] = false_positive_rate
+SCORING_FUNCTIONS["tpr"] = true_positive_rate