datapruning 2.1.4__tar.gz → 2.1.6__tar.gz

{datapruning-2.1.4/datapruning.egg-info → datapruning-2.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 2.1.4
+Version: 2.1.6
 Summary: Intelligent dataset optimization for cleaner, smaller training data
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com

{datapruning-2.1.4 → datapruning-2.1.6}/datapruning/algorithms/base.py

@@ -54,53 +54,59 @@ class BaseOptimizer(ABC):
         X: pd.DataFrame,
         y: pd.Series,
         keep_ratio: float,
-        n_random: int = 3,
-        test_size: float = 0.3,
+        n_random: int = 5,
+        n_splits: int = 3,
         seed: int = 0,
     ) -> bool:
-        """Check whether this strategy beats random on an internal holdout.
+        """Check whether this strategy beats random on internal holdouts.
 
-        Splits the data 70/30, trains the strategy on the 70% subset,
-        validates on the 30% holdout, and compares with random subsets.
-        Returns True if the strategy is beneficial.
+        Runs n_splits stratified train/val splits, each with n_random
+        random baseline subsets.  The strategy is beneficial only if its
+        mean accuracy exceeds random mean by at least half the random
+        standard deviation — removing noisy single-split false positives.
         """
-        from sklearn.model_selection import train_test_split
+        from sklearn.model_selection import StratifiedKFold
         from sklearn.linear_model import LogisticRegression
 
         if len(y) < 50:
             return True
-        y_vals = y.values if hasattr(y, 'values') else np.asarray(y)
+        y_vals = y.values if hasattr(y, "values") else np.asarray(y)
         n_unique = len(np.unique(y_vals))
         if n_unique < 2 or n_unique >= len(y_vals) * 0.8:
             return True
 
-        tr_idx, va_idx = train_test_split(
-            np.arange(len(y)), test_size=test_size,
-            random_state=seed, stratify=y.values,
-        )
-        X_tr, y_tr = X.iloc[tr_idx], y.iloc[tr_idx]
-        X_va, y_va = X.iloc[va_idx], y.iloc[va_idx]
-        n_keep = max(1, int(len(tr_idx) * keep_ratio))
+        strat_accs_all = []
+        random_accs_all = []
 
         try:
-            self.fit(X_tr, y_tr)
-            s_keep = self._select_rows(X_tr, y_tr, keep_ratio)
-            m = LogisticRegression(max_iter=300, C=0.1, random_state=seed)
-            m.fit(X_tr.loc[s_keep], y_tr.loc[s_keep])
-            strat_acc = m.score(X_va, y_va)
-
-            rng = np.random.RandomState(seed)
-            random_accs = []
-            for s in range(n_random):
-                r_idx = rng.choice(len(tr_idx), n_keep, replace=False)
-                r_keep = X_tr.index[r_idx]
-                m2 = LogisticRegression(max_iter=300, C=0.1, random_state=seed)
-                m2.fit(X_tr.loc[r_keep], y_tr.loc[r_keep])
-                random_accs.append(m2.score(X_va, y_va))
-
-            return strat_acc >= np.mean(random_accs)
+            skf = StratifiedKFold(
+                n_splits=n_splits, shuffle=True, random_state=seed
+            )
+            for tr_idx, va_idx in skf.split(np.arange(len(y)), y.values):
+                X_tr, y_tr = X.iloc[tr_idx], y.iloc[tr_idx]
+                X_va, y_va = X.iloc[va_idx], y.iloc[va_idx]
+                n_keep = max(1, int(len(tr_idx) * keep_ratio))
+
+                self.fit(X_tr, y_tr)
+                s_keep = self._select_rows(X_tr, y_tr, keep_ratio)
+                m = LogisticRegression(max_iter=300, C=0.1, random_state=seed)
+                m.fit(X_tr.loc[s_keep], y_tr.loc[s_keep])
+                strat_accs_all.append(m.score(X_va, y_va))
+
+                rng = np.random.RandomState(seed + len(strat_accs_all))
+                for _ in range(n_random):
+                    r_idx = rng.choice(len(tr_idx), n_keep, replace=False)
+                    r_keep = X_tr.index[r_idx]
+                    m2 = LogisticRegression(max_iter=300, C=0.1, random_state=seed)
+                    m2.fit(X_tr.loc[r_keep], y_tr.loc[r_keep])
+                    random_accs_all.append(m2.score(X_va, y_va))
+
+            mean_strat = np.mean(strat_accs_all)
+            mean_random = np.mean(random_accs_all)
+            std_random = np.std(random_accs_all)
+            return mean_strat > mean_random + std_random * 0.5
         except Exception:
-            return True
+            return False
 
     def _random_select(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float, seed: int = 42) -> pd.Index:
         n_target = max(1, int(len(y) * keep_ratio))
@@ -131,9 +137,27 @@ class BaseOptimizer(ABC):
 
         beneficial = self._beat_random_gate(X, y, keep_ratio)
         if not beneficial:
-            keep_idx = self._random_select(X, y, keep_ratio)
-            X_res, y_res = X.loc[keep_idx], y.loc[keep_idx]
-            strategy_method = "random (fallback)"
+            if keep_ratio < 0.25 and self.key != "kcenter":
+                try:
+                    from datapruning.algorithms.kcenter_select import KCenterSelectOptimizer
+                    kcenter = KCenterSelectOptimizer()
+                    kcenter_beneficial = kcenter._beat_random_gate(X, y, keep_ratio)
+                    if kcenter_beneficial:
+                        kcenter.fit(X, y)
+                        X_res, y_res = kcenter.transform(X, y, keep_ratio)
+                        strategy_method = "Diversity Selection (auto)"
+                    else:
+                        keep_idx = self._random_select(X, y, keep_ratio)
+                        X_res, y_res = X.loc[keep_idx], y.loc[keep_idx]
+                        strategy_method = "random (fallback)"
+                except Exception:
+                    keep_idx = self._random_select(X, y, keep_ratio)
+                    X_res, y_res = X.loc[keep_idx], y.loc[keep_idx]
+                    strategy_method = "random (fallback)"
+            else:
+                keep_idx = self._random_select(X, y, keep_ratio)
+                X_res, y_res = X.loc[keep_idx], y.loc[keep_idx]
+                strategy_method = "random (fallback)"
         else:
             strategy_method = self.display_name
 

{datapruning-2.1.4 → datapruning-2.1.6}/datapruning/algorithms/margin_scorer.py

@@ -43,9 +43,9 @@ class MarginScorerOptimizer(BaseOptimizer):
         n_total_target = max(1, int(len(y) * keep_ratio))
         n_minority = min(len(minority_idx), int(n_total_target * 0.8))
         n_majority_target = max(n_total_target - n_minority, 0)
-        majority_scores = self._scores.loc[majority_idx].sort_values(ascending=True)
+        majority_scores = self._scores.loc[majority_idx].sort_values(ascending=False)
         kept_majority_idx = majority_scores.index[:n_majority_target]
-        kept_minority_idx = minority_idx[:n_minority]
+        kept_minority_idx = self._scores.loc[minority_idx].sort_values(ascending=False).index[:n_minority]
         return kept_minority_idx.append(kept_majority_idx)
 
     def why_selected(self) -> str:

{datapruning-2.1.4 → datapruning-2.1.6}/datapruning/algorithms/sliding_window.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 import numpy as np
 import pandas as pd
 from datapruning.algorithms.oof_utils import compute_oof_logits, logits_to_el2n
-from datapruning.research_pruning.selection import sliding_window_select
 from datapruning.algorithms.base import BaseOptimizer
 from datapruning.algorithms.registry import register
 
@@ -32,29 +31,45 @@ class SlidingWindowOptimizer(BaseOptimizer):
     def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
         if self._scores is None:
             self.fit(X, y)
+
         y_arr = y.values
         classes = np.unique(y_arr)
-        low_pct = max(0, (1 - keep_ratio) * 100 / 2)
-        high_pct = min(100, 100 - low_pct)
+        n = len(y_arr)
+        n_target = max(1, int(n * keep_ratio))
+
         if len(classes) < 2:
-            n_target = max(1, int(len(y) * keep_ratio))
             return y.sample(n=n_target, random_state=42).index
-        selected = sliding_window_select(
-            self._scores,
-            low_percentile=low_pct,
-            high_percentile=high_pct,
-            labels=y_arr,
-            preserve_minority=True,
-        )
-        minority_class = y.value_counts().idxmin()
-        minority_idx = y[y == minority_class].index
-        selected_idx = X.index[selected]
-        all_idx = selected_idx.union(minority_idx).unique()
-        n_target = max(1, int(len(y) * keep_ratio))
-        if len(all_idx) > n_target:
-            scores_at_idx = pd.Series(self._scores, index=X.index).loc[all_idx]
-            all_idx = scores_at_idx.sort_values(ascending=True).index[:n_target]
-        return all_idx
+
+        scores = self._scores
+        order = np.argsort(scores)
+        drop_n = max(1, int(n * 0.10))
+        pool = order[:n - drop_n]
+
+        floor_per_class = max(1, int(n_target * 0.5 / len(classes)))
+        selected = set()
+
+        for c in classes:
+            c_mask = y_arr[pool] == c
+            c_indices = pool[c_mask]
+            if len(c_indices) == 0:
+                c_all = np.where(y_arr == c)[0]
+                n_pick = min(floor_per_class, len(c_all))
+                picks = c_all[np.argsort(scores[c_all])[-n_pick:]]
+                for idx in picks:
+                    selected.add(int(idx))
+            else:
+                sorted_c = c_indices[np.argsort(scores[c_indices])]
+                n_pick = min(floor_per_class, len(sorted_c))
+                for idx in sorted_c[-n_pick:]:
+                    selected.add(int(idx))
+
+        for idx in order[::-1]:
+            if len(selected) >= n_target:
+                break
+            selected.add(int(idx))
+
+        selected_arr = np.array(sorted(selected)[:n_target], dtype=int)
+        return X.index[selected_arr]
 
     def why_selected(self) -> str:
         return ("Filters out both the easiest (redundant) and hardest (potentially mislabeled) "

{datapruning-2.1.4 → datapruning-2.1.6/datapruning.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 2.1.4
+Version: 2.1.6
 Summary: Intelligent dataset optimization for cleaner, smaller training data
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com

{datapruning-2.1.4 → datapruning-2.1.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datapruning"
-version = "2.1.4"
+version = "2.1.6"
 description = "Intelligent dataset optimization for cleaner, smaller training data"
 readme = "README.md"
 requires-python = ">=3.10"