classifier-toolkit 0.4.0__tar.gz → 0.5.0__tar.gz

{classifier_toolkit-0.4.0 → classifier_toolkit-0.5.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.5
 Name: classifier-toolkit
-Version: 0.4.0
+Version: 0.5.0
 Project-URL: Documentation, https://supreme-adventure-jg5qkyr.pages.github.io/
 Author-email: "senih.yilmaz" <senih.yilmaz@qonto.com>, "jeremy.fraoua" <jeremy.fraoua@qonto.com>, "gauthier.marquand" <gauthier.marquand@qonto.com>, "arnaud.alepee" <arnaud.alepee@qonto.com>
 License-File: LICENSE
-Requires-Python: <3.14,>=3.9
+Requires-Python: <3.14,>=3.10
 Requires-Dist: catboost<2.0.0,>=1.2.2
 Requires-Dist: category-encoders<3.0.0,>=2.6.3
 Requires-Dist: colorama<0.5.0,>=0.4.6

{classifier_toolkit-0.4.0 → classifier_toolkit-0.5.0}/classifier_toolkit/data_partition/split_train_test.py

@@ -11,200 +11,121 @@ def split_data_recent_orgs(
     date_col="observation_date",
 ):
     """
-    Split data into train and test sets based on the most recent organizations.
-    Takes the most recent organizations (by first observation date) for the test set
-    based on the specified test_size proportion, and the remaining organizations for training.
-    Iteratively adjusts the split to maintain default rate balance within tolerance.
+    Split data into train and test sets on the most recent entities.
+
+    Entities (``org_id_col``) are ordered by their first observation date, and
+    the most recent ones go to the test set, so the test set mimics entities
+    the model will meet after deployment. The number of test entities is
+    ``test_size`` of all entities, adjusted as little as possible so that the
+    train and test event rates differ by at most ``rate_tolerance``: sizes are
+    tried from the target outwards (n, n-1, n+1, n-2, ...) within +/-30% of
+    it (at least +/-5), and the first one within tolerance is kept. If none
+    is, the size with the smallest event-rate difference is used. The target
+    is ``round(test_size * n_entities)``, at least 1.
 
     Parameters:
     -----------
     df : pandas.DataFrame
         The input dataframe
     test_size : float, default=0.2
-        Proportion of organizations to include in test set (most recent ones)
+        Share of **entities** (not rows) put in the test set. Recent entities
+        usually have fewer observations, so the test share of rows is lower.
     rate_tolerance : float, default=0.05
-        Maximum acceptable difference in default rates between train and test (5%)
-    max_iterations : int, default=50
-        Maximum number of iterations to find a balanced split
+        Maximum absolute difference between the train and test event rates
+        (``0.05`` = 5 percentage points; use a value in proportion to the
+        event rate).
     label_col : str, default='default_label'
-        Column containing the binary labels
+        Column containing the binary (0/1) target
     org_id_col : str, default='org_id'
-        Column containing organization IDs
+        Column containing the entity IDs. Entities that share a first
+        observation date are ordered by ID, so the IDs must be mutually
+        comparable (not a mix of ``int`` and ``str``).
     date_col : str, default='observation_date'
         Column containing the observation dates
 
     Returns:
     --------
     tuple
-        (train_df, test_df) - Training and testing dataframes
+        ``(train_df, test_df, stats)``, where ``stats`` is the same dict as
+        the other splitters return (``train_orgs``, ``test_orgs``,
+        ``train_records``, ``test_records``, ``train_target_rate``,
+        ``test_target_rate``, ``target_rate_diff``, ...).
     """
-    # Ensure date column is datetime
     df = df.copy()
     if not pd.api.types.is_datetime64_dtype(df[date_col]):
         df[date_col] = pd.to_datetime(df[date_col])
 
-    # Get the earliest observation date for each organization
+    # First observation date of each entity, oldest first. The entity ID
+    # breaks ties, so which entities of a same-date group end up in test is
+    # deterministic.
     org_first_dates = (
         df.groupby(org_id_col)[date_col]
         .min()
         .reset_index()
-        .sort_values(date_col, ascending=True)
+        .sort_values([date_col, org_id_col], kind="stable")
     )
-
-    # Calculate number of organizations for test set
     total_orgs = len(org_first_dates)
-    n_test_orgs = int(total_orgs * test_size)
-
-    print("=== RECENT ORGANIZATIONS SPLIT WITH RATE TOLERANCE ===")
-    print(f"Test size target: {test_size:.1%}")
-    print(f"Rate tolerance: {rate_tolerance:.1%}")
-    print(f"Total organizations: {total_orgs}")
+    if total_orgs < 2:
+        raise ValueError(
+            f"split_data_recent_orgs needs at least 2 entities in {org_id_col!r}, "
+            f"got {total_orgs}."
+        )
+    n_test_orgs = max(1, round(total_orgs * test_size))
 
-    # Try different splits around the target to find one within tolerance
-    best_split = None
-    best_rate_diff = float("inf")
+    print("=== RECENT ENTITIES SPLIT WITH RATE TOLERANCE ===")
+    print(f"Test size target: {test_size:.1%} of {total_orgs} entities")
+    print(f"Rate tolerance: {rate_tolerance:.2%}")
 
-    # Search range: ±30% of target test size
+    # Sizes from the target outwards, so the first one within tolerance is
+    # the closest to the requested test_size.
     search_range = max(5, int(n_test_orgs * 0.3))
-    min_test_orgs = max(1, n_test_orgs - search_range)
-    max_test_orgs = min(total_orgs - 1, n_test_orgs + search_range)
-
-    print(
-        f"Searching for balanced split (testing {min_test_orgs} to {max_test_orgs} orgs)..."
-    )
-
-    for test_org_count in range(min_test_orgs, max_test_orgs + 1):
-        # Take organizations for this test size
-        test_orgs_data = org_first_dates.tail(test_org_count)
-
-        # Create temporary splits
-        temp_train = df[~df[org_id_col].isin(test_orgs_data[org_id_col].values)]
-        temp_test = df[df[org_id_col].isin(test_orgs_data[org_id_col].values)]
-
-        if len(temp_train) == 0 or len(temp_test) == 0:
+    low = max(1, n_test_orgs - search_range)
+    high = min(total_orgs - 1, n_test_orgs + search_range)
+    candidates = sorted(range(low, high + 1), key=lambda n: (abs(n - n_test_orgs), n))
+
+    best_count, best_rate_diff = None, float("inf")
+    for test_org_count in candidates:
+        test_ids = org_first_dates[org_id_col].tail(test_org_count).to_numpy()
+        is_test = df[org_id_col].isin(test_ids)
+        if is_test.all() or not is_test.any():
             continue
-
-        # Calculate default rates
-        train_rate = temp_train[label_col].mean()
-        test_rate = temp_test[label_col].mean()
-        rate_diff = abs(train_rate - test_rate)
-
-        # Check if this split is better
+        rate_diff = abs(
+            df.loc[~is_test, label_col].mean() - df.loc[is_test, label_col].mean()
+        )
         if rate_diff < best_rate_diff:
-            best_rate_diff = rate_diff
-            best_split = {
-                "test_orgs_data": test_orgs_data,
-                "train_rate": train_rate,
-                "test_rate": test_rate,
-                "rate_diff": rate_diff,
-                "test_org_count": test_org_count,
-                "actual_test_size": len(temp_test) / len(df),
-            }
-
-        # Stop if we found an acceptable split
+            best_count, best_rate_diff = test_org_count, rate_diff
         if rate_diff <= rate_tolerance:
             print(
-                f"✓ Found balanced split at {test_org_count} test orgs (rate diff: {rate_diff:.4f})"
+                f"Found a balanced split at {test_org_count} test entities "
+                f"(event-rate difference: {rate_diff:.4f})"
             )
             break
+    else:
+        if best_count is None:
+            best_count = n_test_orgs
+            print("Could not find any valid split; using the target size.")
+        else:
+            print(
+                f"Best achievable event-rate difference: {best_rate_diff:.4f} "
+                f"(exceeds the tolerance {rate_tolerance:.4f}); using "
+                f"{best_count} test entities."
+            )
 
-    if best_split is None:
-        # Fallback to original target
-        best_split = {
-            "test_orgs_data": org_first_dates.tail(n_test_orgs),
-            "test_org_count": n_test_orgs,
-        }
-        print("⚠️  Could not find any valid split. Using target split.")
-    elif best_rate_diff > rate_tolerance:
-        print(
-            f"⚠️  Best achievable rate difference: {best_rate_diff:.4f} (exceeds tolerance: {rate_tolerance:.4f})"
-        )
-
-    # Use the best split found
-    test_orgs_data = best_split["test_orgs_data"]
-
-    # Split data into train and test
-    train_df = df[~df[org_id_col].isin(test_orgs_data[org_id_col].values)].copy()
-    test_df = df[df[org_id_col].isin(test_orgs_data[org_id_col].values)].copy()
-
-    # Calculate statistics
-    total_orgs = df[org_id_col].nunique()
-    total_records = len(df)
-    total_pos = df[df[label_col] == 1].shape[0]
-    total_neg = df[df[label_col] == 0].shape[0]
-
-    train_orgs = train_df[org_id_col].nunique()
-    train_records = len(train_df)
-    train_pos = train_df[train_df[label_col] == 1].shape[0]
-    train_neg = train_df[train_df[label_col] == 0].shape[0]
-
-    test_orgs = test_df[org_id_col].nunique()
-    test_records = len(test_df)
-    test_pos = test_df[test_df[label_col] == 1].shape[0]
-    test_neg = test_df[test_df[label_col] == 0].shape[0]
-
-    # Verify no overlap
-    train_org_set = set(train_df[org_id_col].unique())
-    test_org_set = set(test_df[org_id_col].unique())
-    overlap = train_org_set.intersection(test_org_set)
-
-    # Print statistics
-    print("\n=== FINAL SPLIT STATISTICS ===")
-    print(f"Test organizations: {test_orgs} ({test_orgs / total_orgs:.1%})")
-    print(f"Train organizations: {train_orgs} ({train_orgs / total_orgs:.1%})")
-
-    print(f"\nTotal records: {total_records}")
-    print(f"Train records: {train_records} ({train_records / total_records:.2%})")
-    print(f"Test records: {test_records} ({test_records / total_records:.2%})")
-
-    print("\nPositive samples:")
-    print(f"Total: {total_pos}")
-    print(f"Train: {train_pos} ({train_pos / total_pos:.2%})")
-    print(f"Test: {test_pos} ({test_pos / total_pos:.2%})")
-
-    print("\nNegative samples:")
-    print(f"Total: {total_neg}")
-    print(f"Train: {train_neg} ({train_neg / total_neg:.2%})")
-    print(f"Test: {test_neg} ({test_neg / total_neg:.2%})")
-
-    print("\nDefault rates:")
-    print(f"Overall: {df[label_col].mean():.4f}")
-    print(f"Train: {train_df[label_col].mean():.4f}")
-    print(f"Test: {test_df[label_col].mean():.4f}")
-    print(
-        f"Difference: {abs(train_df[label_col].mean() - test_df[label_col].mean()):.4f}"
-    )
-
-    print(f"\nOrganization overlap: {len(overlap)} organizations (should be 0)")
+    test_orgs_data = org_first_dates.tail(best_count)
+    is_test = df[org_id_col].isin(test_orgs_data[org_id_col].to_numpy())
+    train_df = df[~is_test].copy()
+    test_df = df[is_test].copy()
 
-    print("\nDate ranges:")
-    print(
-        "Cut-off date for splitting (earliest test org first loan):",
-        test_orgs_data[date_col].min().strftime("%Y-%m-%d"),
-    )
-    print(
-        f"Training orgs (first loan): {train_df.groupby(org_id_col)[date_col].min().min().strftime('%Y-%m-%d')} to {train_df.groupby(org_id_col)[date_col].min().max().strftime('%Y-%m-%d')}"
-    )
-    print(
-        f"Testing orgs (first loan): {test_df.groupby(org_id_col)[date_col].min().min().strftime('%Y-%m-%d')} to {test_df.groupby(org_id_col)[date_col].min().max().strftime('%Y-%m-%d')}"
-    )
-    print(
-        f"Training all observations: {train_df[date_col].min().strftime('%Y-%m-%d')} to {train_df[date_col].max().strftime('%Y-%m-%d')}"
+    stats = _calculate_split_statistics(
+        df, train_df, test_df, label_col, org_id_col, date_col
     )
+    overlap = set(train_df[org_id_col]) & set(test_df[org_id_col])
+    print(f"\nEntity overlap: {len(overlap)} (should be 0)")
     print(
-        f"Testing all observations: {test_df[date_col].min().strftime('%Y-%m-%d')} to {test_df[date_col].max().strftime('%Y-%m-%d')}"
-    )
-
-    return (
-        train_df,
-        test_df,
-        (
-            train_df.shape,
-            test_df.shape,
-            train_df[train_df[label_col] == 1].shape,
-            test_df[test_df[label_col] == 1].shape,
-        ),
+        "Cut-off (first observation of the earliest test entity): "
+        f"{test_orgs_data[date_col].min():%Y-%m-%d}"
     )
+    return train_df, test_df, stats
 
 
 def split_data_stratified_by_org(

{classifier_toolkit-0.4.0 → classifier_toolkit-0.5.0}/classifier_toolkit/explainability/interactions.py

@@ -2,7 +2,7 @@
 
 import math
 from collections.abc import Sequence
-from typing import Union
+from typing import Optional, Union
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -45,28 +45,47 @@ def analyze_shap_interactions(
     shap_values: np.ndarray,
     n_important_features: int = 2,
     class_index: int = -1,
+    features: Optional[Sequence[str]] = None,
 ) -> plt.Figure:
     """Grid of main effects (diagonal) and pairwise interactions (off-diagonal).
 
+    The grid shows the `n_important_features` features with the highest mean
+    absolute SHAP value (from `shap_values`), most important first, or the
+    explicit `features` when given. It used to show the first features of
+    `feature_names`, whatever their importance.
+
     ``class_index`` selects which class's interaction values to plot when the
     model's SHAP output carries a per-class axis; ``shap_values`` must be the
-    matching 2D slice for the same class.
+    matching 2D slice for the same class, with one column per
+    `feature_names` entry.
     """
+    feature_list = list(feature_names)
+    if features is not None:
+        unknown = [f for f in features if f not in feature_list]
+        if unknown:
+            raise ValueError(f"features not in feature_names: {unknown}")
+        important_features = list(features)
+    else:
+        values = np.asarray(shap_values)
+        if values.ndim != 2 or values.shape[1] != len(feature_list):
+            raise ValueError(
+                "shap_values must be 2D with one column per feature_names entry, "
+                f"got shape {values.shape} for {len(feature_list)} features."
+            )
+        ranking = np.argsort(-np.abs(values).mean(axis=0), kind="stable")
+        important_features = [feature_list[i] for i in ranking[:n_important_features]]
+    n_features = len(important_features)
+
     interaction_values = compute_shap_interaction_values(
         model, X_test, feature_names, class_index=class_index
     )
 
-    important_features = list(feature_names)[:n_important_features]
-    n_features = len(important_features)
-
-    fig, axes = plt.subplots(
-        n_features, n_features, figsize=(10 * n_features, 10 * n_features)
-    )
+    side = 5 * n_features  # 5 inches per panel
+    fig, axes = plt.subplots(n_features, n_features, figsize=(side, side))
 
     if n_features == 1:
         axes = np.array([[axes]])
 
-    feature_list = list(feature_names)
     for i in range(n_features):
         for j in range(n_features):
             feature_i = important_features[i]
@@ -111,6 +130,10 @@ def plot_top_shap_interactions(
 ) -> plt.Figure:
     """Plot the strongest feature interactions by mean absolute SHAP interaction.
 
+    Each unordered pair appears once: the interaction matrix is symmetric, so
+    only its upper triangle is ranked (it used to rank both triangles, so
+    every pair was plotted twice).
+
     ``class_index`` selects which class's interaction values to rank and plot
     when the model's SHAP output carries a per-class axis.
     """
@@ -122,18 +145,15 @@ def plot_top_shap_interactions(
     )
 
     interaction_strength = np.abs(interaction_values).mean(axis=0)
-    np.fill_diagonal(interaction_strength, 0)
-
-    flat_indices = np.argsort(interaction_strength.flatten())[-n_top_interactions:]
-    feature_pairs: list[tuple] = []
     names = list(feature_names)
 
-    for flat_idx in flat_indices[::-1]:
-        i, j = np.unravel_index(flat_idx, interaction_strength.shape)
-        if i < j:
-            feature_pairs.append((names[i], names[j], interaction_strength[i, j]))
-        else:
-            feature_pairs.append((names[j], names[i], interaction_strength[i, j]))
+    # Symmetric matrix: rank each unordered pair once (upper triangle).
+    rows, cols = np.triu_indices(len(names), k=1)
+    strengths = interaction_strength[rows, cols]
+    order = np.argsort(-strengths, kind="stable")[:n_top_interactions]
+    feature_pairs: list[tuple] = [
+        (names[rows[k]], names[cols[k]], float(strengths[k])) for k in order
+    ]
 
     n_cols = 2
     n_rows = math.ceil(len(feature_pairs) / n_cols)

{classifier_toolkit-0.4.0 → classifier_toolkit-0.5.0}/classifier_toolkit/explainability/toolkit.py

@@ -60,6 +60,15 @@ class ExplainabilityToolkit:
         self.feature_names = list(feature_names)
         self.numerical_features = list(numerical_features or [])
         self.categorical_features = list(categorical_features or [])
+        unknown = [
+            f
+            for f in self.numerical_features + self.categorical_features
+            if f not in self.feature_names
+        ]
+        if unknown:
+            raise ValueError(
+                f"numerical/categorical features not in feature_names: {unknown}"
+            )
         self.explainer = TreeSHAPExplainer(
             model,
             cat_features=cat_features or categorical_features,
@@ -128,13 +137,14 @@ class ExplainabilityToolkit:
         if not self.numerical_features:
             return None
         self._require_pos_label_if_multiclass("plot_numerical_dependence")
-        n_num = len(self.numerical_features)
         values_2d = (
             self.shap_result_.values_for_label(self.pos_label)
             if self.shap_result_.is_multiclass
             else self.shap_result_.values_for_class()
         )
-        shap_subset = values_2d[:, :n_num]
+        # Select the SHAP columns by name: feature_names need not list the
+        # numerical features first.
+        shap_subset = values_2d[:, self._columns(self.numerical_features)]
         X_num = self.shap_result_.data[self.numerical_features]
         return plot_shap_dependence_grid(
             feature_list=self.numerical_features,
@@ -156,13 +166,12 @@ class ExplainabilityToolkit:
         if not self.categorical_features:
             return None
         self._require_pos_label_if_multiclass("plot_categorical_dependence")
-        n_num = len(self.numerical_features)
         values_2d = (
             self.shap_result_.values_for_label(self.pos_label)
             if self.shap_result_.is_multiclass
             else self.shap_result_.values_for_class()
         )
-        shap_cat = values_2d[:, n_num:]
+        shap_cat = values_2d[:, self._columns(self.categorical_features)]
         X_cat = self.shap_result_.data[self.categorical_features]
         return plot_shap_dependence_grid(
             feature_list=self.categorical_features,
@@ -181,6 +190,10 @@ class ExplainabilityToolkit:
             return self.shap_result_.class_index_for_label(self.pos_label)
         return -1
 
+    def _columns(self, features: Sequence[str]) -> list[int]:
+        """Positions of `features` in the SHAP matrix (``feature_names`` order)."""
+        return [self.feature_names.index(f) for f in features]
+
     def plot_interaction_grid(self, n_important_features: int = 2):
         """Main-effect / interaction grid for top features.