datapruning 2.1.3__tar.gz → 2.1.5__tar.gz

{datapruning-2.1.3/datapruning.egg-info → datapruning-2.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 2.1.3
+Version: 2.1.5
 Summary: Intelligent dataset optimization for cleaner, smaller training data
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com
@@ -104,12 +104,14 @@ DataPruning selects the best algorithm automatically based on dataset characteri
 
 | Method | What it does | Best for |
 |--------|-------------|----------|
-| **Margin Optimization** | Analyzes classification confidence, keeps boundary samples | General tabular data |
-| **Importance Scoring** | Measures sample difficulty via feature distances | Quick baseline, any dataset |
+| **Margin Optimization** | Analyzes classification confidence gap via cross-validation, keeps boundary samples | General tabular data |
+| **Importance Scoring** | Measures sample difficulty via model prediction confidence across CV folds | Quick baseline, any dataset |
 | **Diversity Selection** | Picks the most representative subset (K-Center) | High-dimensional data |
-| **Noise Filtering** | Drops redundant and potentially mislabeled samples | Noisy/messy data |
+| **Noise Filtering** | Drops redundant and potentially mislabeled samples via CV-based difficulty | Noisy/messy data |
 | **Full Pipeline** | Combines all signals for maximum quality | Complex datasets |
 
+All scoring methods use **out-of-fold (OOF) model logits** — each sample is scored by a model that never saw it during training, producing unbiased difficulty estimates. The optimizer also includes a **"must beat random" gate** that validates the selected strategy actually improves over random subsampling on an internal holdout; if not, it safely falls back to random selection.
+
 ## Limits
 
 - Minimum: 1,000 rows (tabular) or 10 images

{datapruning-2.1.3 → datapruning-2.1.5}/README.md

@@ -74,12 +74,14 @@ DataPruning selects the best algorithm automatically based on dataset characteri
 
 | Method | What it does | Best for |
 |--------|-------------|----------|
-| **Margin Optimization** | Analyzes classification confidence, keeps boundary samples | General tabular data |
-| **Importance Scoring** | Measures sample difficulty via feature distances | Quick baseline, any dataset |
+| **Margin Optimization** | Analyzes classification confidence gap via cross-validation, keeps boundary samples | General tabular data |
+| **Importance Scoring** | Measures sample difficulty via model prediction confidence across CV folds | Quick baseline, any dataset |
 | **Diversity Selection** | Picks the most representative subset (K-Center) | High-dimensional data |
-| **Noise Filtering** | Drops redundant and potentially mislabeled samples | Noisy/messy data |
+| **Noise Filtering** | Drops redundant and potentially mislabeled samples via CV-based difficulty | Noisy/messy data |
 | **Full Pipeline** | Combines all signals for maximum quality | Complex datasets |
 
+All scoring methods use **out-of-fold (OOF) model logits** — each sample is scored by a model that never saw it during training, producing unbiased difficulty estimates. The optimizer also includes a **"must beat random" gate** that validates the selected strategy actually improves over random subsampling on an internal holdout; if not, it safely falls back to random selection.
+
 ## Limits
 
 - Minimum: 1,000 rows (tabular) or 10 images

{datapruning-2.1.3 → datapruning-2.1.5}/datapruning/algorithms/base.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
 from typing import Optional
+import numpy as np
 import pandas as pd
 
 
@@ -15,6 +16,8 @@ class OptimizationResult:
     minority_count_before: Optional[int]
     minority_count_after: Optional[int]
     runtime_seconds: float
+    beneficial: bool = True
+    strategy_method: str = ""
 
 
 @dataclass
@@ -27,6 +30,7 @@ class Explanation:
     estimated_training_gain_pct: float
     memory_saved_pct: float
     limitations: list[str] = field(default_factory=list)
+    beneficial: bool = True
 
 
 class BaseOptimizer(ABC):
@@ -45,18 +49,83 @@ class BaseOptimizer(ABC):
         keep_idx = self._select_rows(X, y, keep_ratio)
         return X.loc[keep_idx], y.loc[keep_idx]
 
+    def _beat_random_gate(
+        self,
+        X: pd.DataFrame,
+        y: pd.Series,
+        keep_ratio: float,
+        n_random: int = 5,
+        n_splits: int = 3,
+        seed: int = 0,
+    ) -> bool:
+        """Check whether this strategy beats random on internal holdouts.
+
+        Runs n_splits stratified train/val splits, each with n_random
+        random baseline subsets.  The strategy is beneficial only if its
+        mean accuracy exceeds random mean by at least half the random
+        standard deviation — removing noisy single-split false positives.
+        """
+        from sklearn.model_selection import StratifiedKFold
+        from sklearn.linear_model import LogisticRegression
+
+        if len(y) < 50:
+            return True
+        y_vals = y.values if hasattr(y, "values") else np.asarray(y)
+        n_unique = len(np.unique(y_vals))
+        if n_unique < 2 or n_unique >= len(y_vals) * 0.8:
+            return True
+
+        strat_accs_all = []
+        random_accs_all = []
+
+        try:
+            skf = StratifiedKFold(
+                n_splits=n_splits, shuffle=True, random_state=seed
+            )
+            for tr_idx, va_idx in skf.split(np.arange(len(y)), y.values):
+                X_tr, y_tr = X.iloc[tr_idx], y.iloc[tr_idx]
+                X_va, y_va = X.iloc[va_idx], y.iloc[va_idx]
+                n_keep = max(1, int(len(tr_idx) * keep_ratio))
+
+                self.fit(X_tr, y_tr)
+                s_keep = self._select_rows(X_tr, y_tr, keep_ratio)
+                m = LogisticRegression(max_iter=300, C=0.1, random_state=seed)
+                m.fit(X_tr.loc[s_keep], y_tr.loc[s_keep])
+                strat_accs_all.append(m.score(X_va, y_va))
+
+                rng = np.random.RandomState(seed + len(strat_accs_all))
+                for _ in range(n_random):
+                    r_idx = rng.choice(len(tr_idx), n_keep, replace=False)
+                    r_keep = X_tr.index[r_idx]
+                    m2 = LogisticRegression(max_iter=300, C=0.1, random_state=seed)
+                    m2.fit(X_tr.loc[r_keep], y_tr.loc[r_keep])
+                    random_accs_all.append(m2.score(X_va, y_va))
+
+            mean_strat = np.mean(strat_accs_all)
+            mean_random = np.mean(random_accs_all)
+            std_random = np.std(random_accs_all)
+            return mean_strat > mean_random + std_random * 0.5
+        except Exception:
+            return True
+
+    def _random_select(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float, seed: int = 42) -> pd.Index:
+        n_target = max(1, int(len(y) * keep_ratio))
+        rng = np.random.RandomState(seed)
+        indices = rng.choice(len(y), n_target, replace=False)
+        return X.index[indices]
+
     def optimize(self, df: pd.DataFrame, target_col: str, keep_ratio: float = 0.5) -> OptimizationResult:
         import time
-        import numpy as np
-        from sklearn.preprocessing import LabelEncoder
 
         X, y = df.drop(columns=[target_col]), df[target_col].copy()
 
         for col in X.columns:
             if not np.issubdtype(X[col].dtype, np.number):
+                from sklearn.preprocessing import LabelEncoder
                 X[col] = LabelEncoder().fit_transform(X[col].astype(str))
 
         if not np.issubdtype(y.dtype, np.number):
+            from sklearn.preprocessing import LabelEncoder
             y = pd.Series(LabelEncoder().fit_transform(y.astype(str)), index=y.index)
 
         X = X.fillna(0).astype(np.float64)
@@ -65,6 +134,15 @@ class BaseOptimizer(ABC):
         self.fit(X, y)
         X_res, y_res = self.transform(X, y, keep_ratio)
         runtime = time.time() - t0
+
+        beneficial = self._beat_random_gate(X, y, keep_ratio)
+        if not beneficial:
+            keep_idx = self._random_select(X, y, keep_ratio)
+            X_res, y_res = X.loc[keep_idx], y.loc[keep_idx]
+            strategy_method = "random (fallback)"
+        else:
+            strategy_method = self.display_name
+
         minority_after = int(y_res.value_counts().min()) if y_res.nunique() > 1 else None
         optimized_df = X_res.copy()
         optimized_df[target_col] = y_res.values
@@ -77,6 +155,8 @@ class BaseOptimizer(ABC):
             minority_count_before=minority_before,
             minority_count_after=minority_after,
             runtime_seconds=runtime,
+            beneficial=beneficial,
+            strategy_method=strategy_method,
         )
 
     def explain(self, result: OptimizationResult) -> Explanation:
@@ -88,7 +168,7 @@ class BaseOptimizer(ABC):
                 (result.minority_count_after or 0) / result.minority_count_before * 100, 1
             )
         return Explanation(
-            algorithm=self.display_name,
+            algorithm=result.strategy_method or self.display_name,
             why_selected=self.why_selected(),
             rows_removed=rows_removed,
             reduction_pct=reduction_pct,
@@ -96,6 +176,7 @@ class BaseOptimizer(ABC):
             estimated_training_gain_pct=reduction_pct,
             memory_saved_pct=reduction_pct,
             limitations=self.limitations(),
+            beneficial=result.beneficial,
         )
 
     @staticmethod

{datapruning-2.1.3 → datapruning-2.1.5}/datapruning/algorithms/el2n_scorer.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 import numpy as np
 import pandas as pd
+from datapruning.algorithms.oof_utils import compute_oof_logits, logits_to_el2n
 from datapruning.algorithms.base import BaseOptimizer
 from datapruning.algorithms.registry import register
 
@@ -15,15 +16,24 @@ class EL2NScorerOptimizer(BaseOptimizer):
 
     def fit(self, X: pd.DataFrame, y: pd.Series) -> "EL2NScorerOptimizer":
         X_arr = X.values.astype(np.float64)
-        y_arr = y.values
+        y_arr = y.values.astype(int)
         classes = np.unique(y_arr)
-        scores = np.zeros(len(y_arr), dtype=np.float64)
-        for cls in classes:
-            cls_mask = y_arr == cls
-            centroid = X_arr[cls_mask].mean(axis=0)
-            dists = np.linalg.norm(X_arr - centroid, axis=1)
-            scores = np.maximum(scores, dists)
-        self._scores = pd.Series(scores, index=y.index)
+        if len(classes) < 2:
+            self._scores = pd.Series(np.zeros(len(y_arr)), index=y.index)
+            return self
+        try:
+            logits = compute_oof_logits(X_arr, y_arr)
+            el2n = logits_to_el2n(logits, y_arr)
+            self._scores = pd.Series(el2n, index=y.index)
+        except Exception:
+            centroids = np.zeros((len(classes), X_arr.shape[1]))
+            for i, c in enumerate(classes):
+                centroids[i] = X_arr[y_arr == c].mean(axis=0)
+            scores = np.zeros(len(y_arr), dtype=np.float64)
+            for i, c in enumerate(classes):
+                dists = np.linalg.norm(X_arr - centroids[i], axis=1)
+                scores = np.maximum(scores, dists)
+            self._scores = pd.Series(scores, index=y.index)
         return self
 
     def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
@@ -46,12 +56,13 @@ class EL2NScorerOptimizer(BaseOptimizer):
 
     def why_selected(self) -> str:
         return ("Measures how difficult each sample is for the model to classify by analyzing "
-                "feature-space distances. Keeps the most informative boundary samples that "
-                "help the model learn better decision regions. Works reliably on any dataset.")
+                "model prediction confidence across cross-validation folds. Keeps the most "
+                "informative boundary samples that help the model learn better decision regions. "
+                "Works reliably on any dataset.")
 
     def limitations(self) -> list[str]:
         return [
-            "Uses distance-based approximation — slightly less precise than model-based methods.",
+            "Requires a quick cross-validation step during analysis.",
             "May select outliers on very noisy datasets.",
         ]
 

{datapruning-2.1.3 → datapruning-2.1.5}/datapruning/algorithms/margin_scorer.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 import numpy as np
 import pandas as pd
-from sklearn.linear_model import LogisticRegression
+from datapruning.algorithms.oof_utils import compute_oof_logits, logits_to_margin
 from datapruning.algorithms.base import BaseOptimizer
 from datapruning.algorithms.registry import register
 
@@ -15,18 +15,16 @@ class MarginScorerOptimizer(BaseOptimizer):
         self._scores = None
 
     def fit(self, X: pd.DataFrame, y: pd.Series) -> "MarginScorerOptimizer":
-        X_arr, y_arr = X.values.astype(np.float64), y.values
+        X_arr = X.values.astype(np.float64)
+        y_arr = y.values.astype(int)
         classes = np.unique(y_arr)
         if len(classes) < 2:
             self._scores = pd.Series(np.zeros(len(y_arr)), index=y.index)
             return self
         try:
-            lr = LogisticRegression(max_iter=1000, random_state=42)
-            lr.fit(X_arr, y_arr)
-            probs = lr.predict_proba(X_arr)
-            class_to_col = {c: i for i, c in enumerate(lr.classes_)}
-            true_probs = np.array([probs[i, class_to_col[y_arr[i]]] for i in range(len(y_arr))])
-            self._scores = pd.Series(true_probs, index=y.index)
+            logits = compute_oof_logits(X_arr, y_arr)
+            margin = logits_to_margin(logits, y_arr)
+            self._scores = pd.Series(margin, index=y.index)
         except Exception:
             feat_norms = np.linalg.norm(X_arr, axis=1)
             self._scores = pd.Series(feat_norms, index=y.index)
@@ -45,19 +43,21 @@ class MarginScorerOptimizer(BaseOptimizer):
         n_total_target = max(1, int(len(y) * keep_ratio))
         n_minority = min(len(minority_idx), int(n_total_target * 0.8))
         n_majority_target = max(n_total_target - n_minority, 0)
-        majority_scores = self._scores.loc[majority_idx].sort_values(ascending=True)
+        majority_scores = self._scores.loc[majority_idx].sort_values(ascending=False)
         kept_majority_idx = majority_scores.index[:n_majority_target]
-        kept_minority_idx = minority_idx[:n_minority]
+        kept_minority_idx = self._scores.loc[minority_idx].sort_values(ascending=False).index[:n_minority]
         return kept_minority_idx.append(kept_majority_idx)
 
     def why_selected(self) -> str:
-        return ("Analyzes how confidently each data point is classified, keeping the most "
-                "informative samples near decision boundaries. Automatically preserves all "
-                "minority-class examples to maintain balanced training data.")
+        return ("Analyzes how confidently each data point is classified by looking at "
+                "the gap between the top two predicted classes across cross-validation "
+                "folds. Keeps the most informative samples near decision boundaries. "
+                "Automatically preserves all minority-class examples to maintain "
+                "balanced training data.")
 
     def limitations(self) -> list[str]:
         return [
-            "Requires a quick model fitting step during analysis.",
+            "Requires a quick cross-validation step during analysis.",
             "May be less effective on extremely high-dimensional datasets.",
         ]
 

datapruning-2.1.5/datapruning/algorithms/oof_utils.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+from sklearn.model_selection import StratifiedKFold
+
+
+def compute_oof_logits(
+    X: np.ndarray,
+    y: np.ndarray,
+    n_folds: int = 5,
+    seed: int = 0,
+    C: float = 0.1,
+    max_iter: int = 300,
+) -> np.ndarray:
+    """Compute out-of-fold log-probabilities via stratified K-fold CV.
+
+    Each sample is scored by a model that never saw it during training,
+    producing unbiased per-sample logits.  This is the correct input for
+    EL2N, margin, and sliding-window difficulty scores.
+
+    Returns
+    -------
+    logits : ndarray of shape (n_samples, n_classes)
+        Log-probability of each class for every sample.
+    """
+    X = np.asarray(X, dtype=np.float64)
+    y = np.asarray(y).astype(int)
+    n_classes = len(np.unique(y))
+    out = np.zeros((len(X), n_classes), dtype=np.float64)
+
+    skf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)
+    for tr_idx, va_idx in skf.split(X, y):
+        model = LogisticRegression(
+            max_iter=max_iter, C=C, random_state=seed, solver="lbfgs"
+        )
+        model.fit(X[tr_idx], y[tr_idx])
+        log_proba = model.predict_log_proba(X[va_idx])
+        class_cols = model.classes_.astype(int)
+        out[np.ix_(va_idx, class_cols)] = log_proba
+
+    return out
+
+
+def logits_to_el2n(logits: np.ndarray, y: np.ndarray) -> np.ndarray:
+    """EL2N score from model logits: ||softmax(z) - onehot(y)||.
+
+    Higher values indicate harder / more informative samples.
+    """
+    z = logits - logits.max(axis=1, keepdims=True)
+    p = np.exp(z)
+    p /= p.sum(axis=1, keepdims=True)
+    oh = np.zeros_like(p)
+    oh[np.arange(len(y)), y] = 1.0
+    return np.linalg.norm(p - oh, axis=1)
+
+
+def logits_to_margin(logits: np.ndarray, y: np.ndarray) -> np.ndarray:
+    """Margin score from model logits: confidence gap between top-1 and top-2.
+
+    Lower values indicate harder / more ambiguous samples (near decision
+    boundary).  The score is returned as (1 - margin) so that higher
+    values consistently mean "harder" across all scorers.
+    """
+    z = logits - logits.max(axis=1, keepdims=True)
+    p = np.exp(z)
+    p /= p.sum(axis=1, keepdims=True)
+    n = len(y)
+    margins = np.zeros(n, dtype=np.float64)
+    for i in range(n):
+        sorted_p = np.sort(p[i])[::-1]
+        if len(sorted_p) >= 2:
+            margins[i] = sorted_p[0] - sorted_p[1]
+        else:
+            margins[i] = sorted_p[0]
+    return 1.0 - margins

{datapruning-2.1.3 → datapruning-2.1.5}/datapruning/algorithms/sliding_window.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 import numpy as np
 import pandas as pd
-from sklearn.linear_model import LogisticRegression
+from datapruning.algorithms.oof_utils import compute_oof_logits, logits_to_el2n
 from datapruning.research_pruning.selection import sliding_window_select
 from datapruning.algorithms.base import BaseOptimizer
 from datapruning.algorithms.registry import register
@@ -16,18 +16,15 @@ class SlidingWindowOptimizer(BaseOptimizer):
         self._scores = None
 
     def fit(self, X: pd.DataFrame, y: pd.Series) -> "SlidingWindowOptimizer":
-        X_arr, y_arr = X.values.astype(np.float64), y.values
+        X_arr = X.values.astype(np.float64)
+        y_arr = y.values.astype(int)
         classes = np.unique(y_arr)
         if len(classes) < 2:
             self._scores = np.zeros(len(y_arr))
             return self
         try:
-            lr = LogisticRegression(max_iter=1000, random_state=42)
-            lr.fit(X_arr, y_arr)
-            probs = lr.predict_proba(X_arr)
-            class_to_col = {c: i for i, c in enumerate(lr.classes_)}
-            true_probs = np.array([probs[i, class_to_col[y_arr[i]]] for i in range(len(y_arr))])
-            self._scores = true_probs
+            logits = compute_oof_logits(X_arr, y_arr)
+            self._scores = logits_to_el2n(logits, y_arr)
         except Exception:
             self._scores = np.linalg.norm(X_arr, axis=1)
         return self
@@ -56,17 +53,18 @@ class SlidingWindowOptimizer(BaseOptimizer):
         n_target = max(1, int(len(y) * keep_ratio))
         if len(all_idx) > n_target:
             scores_at_idx = pd.Series(self._scores, index=X.index).loc[all_idx]
-            all_idx = scores_at_idx.sort_values(ascending=True).index[:n_target]
+            all_idx = scores_at_idx.sort_values(ascending=False).index[:n_target]
         return all_idx
 
     def why_selected(self) -> str:
         return ("Filters out both the easiest (redundant) and hardest (potentially mislabeled) "
-                "samples, keeping only the informative middle ground. Best for cleaning "
+                "samples, keeping only the informative middle ground. Uses cross-validation "
+                "to score samples by model prediction difficulty. Best for cleaning "
                 "datasets that may contain labeling errors or noisy data.")
 
     def limitations(self) -> list[str]:
         return [
-            "Requires a quick model fitting step during analysis.",
+            "Requires a quick cross-validation step during analysis.",
             "May occasionally remove genuinely hard but correct samples.",
         ]
 

{datapruning-2.1.3 → datapruning-2.1.5/datapruning.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 2.1.3
+Version: 2.1.5
 Summary: Intelligent dataset optimization for cleaner, smaller training data
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com
@@ -104,12 +104,14 @@ DataPruning selects the best algorithm automatically based on dataset characteri
 
 | Method | What it does | Best for |
 |--------|-------------|----------|
-| **Margin Optimization** | Analyzes classification confidence, keeps boundary samples | General tabular data |
-| **Importance Scoring** | Measures sample difficulty via feature distances | Quick baseline, any dataset |
+| **Margin Optimization** | Analyzes classification confidence gap via cross-validation, keeps boundary samples | General tabular data |
+| **Importance Scoring** | Measures sample difficulty via model prediction confidence across CV folds | Quick baseline, any dataset |
 | **Diversity Selection** | Picks the most representative subset (K-Center) | High-dimensional data |
-| **Noise Filtering** | Drops redundant and potentially mislabeled samples | Noisy/messy data |
+| **Noise Filtering** | Drops redundant and potentially mislabeled samples via CV-based difficulty | Noisy/messy data |
 | **Full Pipeline** | Combines all signals for maximum quality | Complex datasets |
 
+All scoring methods use **out-of-fold (OOF) model logits** — each sample is scored by a model that never saw it during training, producing unbiased difficulty estimates. The optimizer also includes a **"must beat random" gate** that validates the selected strategy actually improves over random subsampling on an internal holdout; if not, it safely falls back to random selection.
+
 ## Limits
 
 - Minimum: 1,000 rows (tabular) or 10 images

{datapruning-2.1.3 → datapruning-2.1.5}/datapruning.egg-info/SOURCES.txt

@@ -18,6 +18,7 @@ datapruning/algorithms/el2n_scorer.py
 datapruning/algorithms/fusion_pipeline.py
 datapruning/algorithms/kcenter_select.py
 datapruning/algorithms/margin_scorer.py
+datapruning/algorithms/oof_utils.py
 datapruning/algorithms/registry.py
 datapruning/algorithms/sliding_window.py
 datapruning/engine/__init__.py

{datapruning-2.1.3 → datapruning-2.1.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datapruning"
-version = "2.1.3"
+version = "2.1.5"
 description = "Intelligent dataset optimization for cleaner, smaller training data"
 readme = "README.md"
 requires-python = ">=3.10"