datapruning 2.1.5__tar.gz → 2.1.6__tar.gz

{datapruning-2.1.5/datapruning.egg-info → datapruning-2.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 2.1.5
+Version: 2.1.6
 Summary: Intelligent dataset optimization for cleaner, smaller training data
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com

{datapruning-2.1.5 → datapruning-2.1.6}/datapruning/algorithms/base.py

@@ -106,7 +106,7 @@ class BaseOptimizer(ABC):
             std_random = np.std(random_accs_all)
             return mean_strat > mean_random + std_random * 0.5
         except Exception:
-            return True
+            return False
 
     def _random_select(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float, seed: int = 42) -> pd.Index:
         n_target = max(1, int(len(y) * keep_ratio))
@@ -137,9 +137,27 @@ class BaseOptimizer(ABC):
 
         beneficial = self._beat_random_gate(X, y, keep_ratio)
         if not beneficial:
-            keep_idx = self._random_select(X, y, keep_ratio)
-            X_res, y_res = X.loc[keep_idx], y.loc[keep_idx]
-            strategy_method = "random (fallback)"
+            if keep_ratio < 0.25 and self.key != "kcenter":
+                try:
+                    from datapruning.algorithms.kcenter_select import KCenterSelectOptimizer
+                    kcenter = KCenterSelectOptimizer()
+                    kcenter_beneficial = kcenter._beat_random_gate(X, y, keep_ratio)
+                    if kcenter_beneficial:
+                        kcenter.fit(X, y)
+                        X_res, y_res = kcenter.transform(X, y, keep_ratio)
+                        strategy_method = "Diversity Selection (auto)"
+                    else:
+                        keep_idx = self._random_select(X, y, keep_ratio)
+                        X_res, y_res = X.loc[keep_idx], y.loc[keep_idx]
+                        strategy_method = "random (fallback)"
+                except Exception:
+                    keep_idx = self._random_select(X, y, keep_ratio)
+                    X_res, y_res = X.loc[keep_idx], y.loc[keep_idx]
+                    strategy_method = "random (fallback)"
+            else:
+                keep_idx = self._random_select(X, y, keep_ratio)
+                X_res, y_res = X.loc[keep_idx], y.loc[keep_idx]
+                strategy_method = "random (fallback)"
         else:
             strategy_method = self.display_name
 

{datapruning-2.1.5 → datapruning-2.1.6}/datapruning/algorithms/sliding_window.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 import numpy as np
 import pandas as pd
 from datapruning.algorithms.oof_utils import compute_oof_logits, logits_to_el2n
-from datapruning.research_pruning.selection import sliding_window_select
 from datapruning.algorithms.base import BaseOptimizer
 from datapruning.algorithms.registry import register
 
@@ -32,29 +31,45 @@ class SlidingWindowOptimizer(BaseOptimizer):
     def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
         if self._scores is None:
             self.fit(X, y)
+
         y_arr = y.values
         classes = np.unique(y_arr)
-        low_pct = max(0, (1 - keep_ratio) * 100 / 2)
-        high_pct = min(100, 100 - low_pct)
+        n = len(y_arr)
+        n_target = max(1, int(n * keep_ratio))
+
         if len(classes) < 2:
-            n_target = max(1, int(len(y) * keep_ratio))
             return y.sample(n=n_target, random_state=42).index
-        selected = sliding_window_select(
-            self._scores,
-            low_percentile=low_pct,
-            high_percentile=high_pct,
-            labels=y_arr,
-            preserve_minority=True,
-        )
-        minority_class = y.value_counts().idxmin()
-        minority_idx = y[y == minority_class].index
-        selected_idx = X.index[selected]
-        all_idx = selected_idx.union(minority_idx).unique()
-        n_target = max(1, int(len(y) * keep_ratio))
-        if len(all_idx) > n_target:
-            scores_at_idx = pd.Series(self._scores, index=X.index).loc[all_idx]
-            all_idx = scores_at_idx.sort_values(ascending=False).index[:n_target]
-        return all_idx
+
+        scores = self._scores
+        order = np.argsort(scores)
+        drop_n = max(1, int(n * 0.10))
+        pool = order[:n - drop_n]
+
+        floor_per_class = max(1, int(n_target * 0.5 / len(classes)))
+        selected = set()
+
+        for c in classes:
+            c_mask = y_arr[pool] == c
+            c_indices = pool[c_mask]
+            if len(c_indices) == 0:
+                c_all = np.where(y_arr == c)[0]
+                n_pick = min(floor_per_class, len(c_all))
+                picks = c_all[np.argsort(scores[c_all])[-n_pick:]]
+                for idx in picks:
+                    selected.add(int(idx))
+            else:
+                sorted_c = c_indices[np.argsort(scores[c_indices])]
+                n_pick = min(floor_per_class, len(sorted_c))
+                for idx in sorted_c[-n_pick:]:
+                    selected.add(int(idx))
+
+        for idx in order[::-1]:
+            if len(selected) >= n_target:
+                break
+            selected.add(int(idx))
+
+        selected_arr = np.array(sorted(selected)[:n_target], dtype=int)
+        return X.index[selected_arr]
 
     def why_selected(self) -> str:
         return ("Filters out both the easiest (redundant) and hardest (potentially mislabeled) "

{datapruning-2.1.5 → datapruning-2.1.6/datapruning.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 2.1.5
+Version: 2.1.6
 Summary: Intelligent dataset optimization for cleaner, smaller training data
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com

{datapruning-2.1.5 → datapruning-2.1.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datapruning"
-version = "2.1.5"
+version = "2.1.6"
 description = "Intelligent dataset optimization for cleaner, smaller training data"
 readme = "README.md"
 requires-python = ">=3.10"