classifier-toolkit 0.2.6__tar.gz → 0.3.0__tar.gz

{classifier_toolkit-0.2.6 → classifier_toolkit-0.3.0}/.github/workflows/checks.yaml

@@ -97,7 +97,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test-group: [eda, feature_selection, feature_reduction]
+        test-group: [eda, feature_selection, feature_reduction, data_partition]
     steps:
       - uses: actions/checkout@v6
 

{classifier_toolkit-0.2.6 → classifier_toolkit-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifier-toolkit
-Version: 0.2.6
+Version: 0.3.0
 Project-URL: Documentation, http://classifier-toolkit.gh-pages.qonto.co/
 Author-email: "senih.yilmaz" <senih.yilmaz@qonto.com>, "jeremy.fraoua" <jeremy.fraoua@qonto.com>, "gauthier.marquand" <gauthier.marquand@qonto.com>, "arnaud.alepee" <arnaud.alepee@qonto.com>
 License-File: LICENSE

classifier_toolkit-0.3.0/classifier_toolkit/data_partition/__init__.py

@@ -0,0 +1,19 @@
+from .data_preprocess import (
+    filter_default_observations_greedy,
+    filter_default_observations_priority,
+)
+from .optimize_data import optimize_dataframe_dtypes
+from .split_train_test import (
+    split_data_by_date,
+    split_data_recent_orgs,
+    split_data_stratified_by_org,
+)
+
+__all__ = [
+    "filter_default_observations_greedy",
+    "filter_default_observations_priority",
+    "optimize_dataframe_dtypes",
+    "split_data_by_date",
+    "split_data_recent_orgs",
+    "split_data_stratified_by_org",
+]

classifier_toolkit-0.3.0/classifier_toolkit/data_partition/data_preprocess.py

@@ -0,0 +1,152 @@
+import pandas as pd
+
+
+def _filter_org_with_dedup(group, date_col, label_col, dedup_fn):
+    group = group.sort_values(date_col)
+    has_default = (group[label_col] == 1).any()
+
+    if not has_default:
+        return dedup_fn(group)
+
+    first_default_date = group.loc[group[label_col] == 1, date_col].min()
+    before = group[group[date_col] < first_default_date]
+    at_first = group[group[date_col] == first_default_date]
+    after = group[(group[date_col] > first_default_date) & (group[label_col] == 1)]
+
+    return pd.concat(
+        [dedup_fn(before), at_first, dedup_fn(after, ref_date=first_default_date)]
+    )
+
+
+def filter_default_observations_priority(
+    df,
+    org_id_col="org_id",
+    date_col="observation_date",
+    label_col="default_label",
+    priority_col="label",
+    priority_value="paylater",
+    min_days=7,
+):
+    """
+    Filter a dataset based on default events, then deduplicate by minimum time gap.
+
+    Two sequential filters are applied per organization:
+
+    1st filter — default event logic:
+        - No default events (default_label=1): keep all records.
+        - Has default events: keep all records before the first default date,
+          and only keep default_label=1 records on/after the first default date.
+
+    2nd filter — deduplication:
+        - Enforce a minimum gap of min_days between consecutive kept rows.
+        - When two rows are within min_days of each other, keep the one whose
+          priority_col equals priority_value ("paylater" beats "SDD").
+          If both or neither have priority, keep the earlier row.
+
+    Parameters:
+    -----------
+    df : pandas.DataFrame
+    org_id_col : str, default='org_id'
+    date_col : str, default='observation_date'
+    label_col : str, default='default_label'
+        Binary default indicator (1=default, 0=no default).
+    priority_col : str, default='label'
+        Column used to break ties during deduplication. Contains "paylater" or "SDD".
+    priority_value : str, default='paylater'
+        Value in priority_col that takes precedence when two rows are within min_days.
+    min_days : int, default=7
+        Minimum number of days required between consecutive kept rows.
+
+    Returns:
+    --------
+    pandas.DataFrame
+    """
+
+    def _dedup(group, ref_date=None):
+        kept_indices = []
+        kept_has_priority = []
+        last_date = ref_date
+
+        for row in group.itertuples():
+            date = getattr(row, date_col)
+            has_priority = getattr(row, priority_col) == priority_value
+
+            if last_date is None or (date - last_date).days >= min_days:
+                kept_indices.append(row.Index)
+                kept_has_priority.append(has_priority)
+                last_date = date
+            elif kept_indices and has_priority and not kept_has_priority[-1]:
+                # Current row wins — replace the last kept row
+                kept_indices[-1] = row.Index
+                kept_has_priority[-1] = True
+                last_date = date
+            # else: keep the already-kept row (same priority or earlier wins)
+
+        return group.loc[kept_indices]
+
+    df_copy = df.copy()
+    df_copy[date_col] = pd.to_datetime(df_copy[date_col])
+
+    return (
+        df_copy.groupby(org_id_col, group_keys=False)
+        .apply(lambda g: _filter_org_with_dedup(g, date_col, label_col, _dedup))
+        .reset_index(drop=True)
+    )
+
+
+def filter_default_observations_greedy(
+    df,
+    org_id_col="org_id",
+    date_col="observation_date",
+    label_col="default_label",
+    min_days=7,
+):
+    """
+    Filter a dataset based on default events with a minimum gap between observations.
+
+    For each organization:
+    - If there are no default events (label=1), keep all records where consecutive
+      observation dates are at least min_days apart.
+    - If there are default events:
+        - Keep records before the first default date, enforcing min_days gap.
+        - Always include the first default date record (ignores min_days rule).
+        - Keep only default=1 records after the first default date, enforcing
+          min_days gap measured from the first default date onward.
+
+    Parameters:
+    -----------
+    df : pandas.DataFrame
+        The input dataframe containing organization data
+    org_id_col : str, default='org_id'
+        Column name for organization identifier
+    date_col : str, default='observation_date'
+        Column name for observation date
+    label_col : str, default='default_label'
+        Column name for default label (1=default, 0=no default)
+    min_days : int, default=7
+        Minimum number of days required between consecutive kept records
+
+    Returns:
+    --------
+    pandas.DataFrame
+        Filtered dataframe according to the specified rules
+    """
+
+    def _dedup(group, ref_date=None):
+        kept_indices = []
+        last_date = ref_date
+        for row in group.itertuples():
+            date = getattr(row, date_col)
+            if last_date is None or (date - last_date).days >= min_days:
+                kept_indices.append(row.Index)
+                last_date = date
+        return group.loc[kept_indices]
+
+    df_copy = df.copy()
+    df_copy[date_col] = pd.to_datetime(df_copy[date_col])
+
+    return (
+        df_copy.groupby(org_id_col, group_keys=False)
+        .apply(lambda g: _filter_org_with_dedup(g, date_col, label_col, _dedup))
+        .reset_index(drop=True)
+    )

classifier_toolkit-0.3.0/classifier_toolkit/data_partition/optimize_data.py

@@ -0,0 +1,47 @@
+from collections import defaultdict
+
+import pandas as pd
+
+
+def optimize_dataframe_dtypes(df, categorical_threshold=0.5, verbose=True):
+    """
+    Optimize DataFrame data types to reduce memory usage.
+
+    Parameters:
+    - df: pandas DataFrame to optimize
+    - categorical_threshold: Convert object columns to category if unique ratio < threshold
+    - verbose: Print optimization details
+
+    Returns:
+    - Optimized DataFrame
+    - Dictionary mapping dtype names to lists of converted columns, plus
+      'memory_before_mb' and 'memory_after_mb' float entries
+    """
+    df_opt = df.copy()
+    optimizations = defaultdict(list)
+
+    for col in df_opt.select_dtypes(include=["int64"]).columns:
+        if df_opt[col].isnull().any():
+            continue
+        downcast = "unsigned" if df_opt[col].min() >= 0 else "integer"
+        df_opt[col] = pd.to_numeric(df_opt[col], downcast=downcast)
+        optimizations[str(df_opt[col].dtype)].append(col)
+
+    n = len(df_opt)
+    for col in df_opt.select_dtypes(include=["object"]).columns:
+        if df_opt[col].nunique() / n < categorical_threshold:
+            df_opt[col] = df_opt[col].astype("category")
+            optimizations["category"].append(col)
+
+    mb_before = df.memory_usage(deep=True).sum() / 1024**2
+    mb_after = df_opt.memory_usage(deep=True).sum() / 1024**2
+    optimizations["memory_before_mb"] = mb_before
+    optimizations["memory_after_mb"] = mb_after
+
+    if verbose:
+        print(f"Memory reduced from {mb_before:.2f} MB to {mb_after:.2f} MB")
+        print(
+            f"Savings: {mb_before - mb_after:.2f} MB ({(mb_before - mb_after) / mb_before * 100:.1f}%)"
+        )
+
+    return df_opt, optimizations