datapruning 2.1.0__tar.gz → 2.1.2__tar.gz

{datapruning-2.1.0/datapruning.egg-info → datapruning-2.1.2}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 2.1.0
-Summary: Intelligent data pruning for ML datasets
+Version: 2.1.2
+Summary: Intelligent dataset optimization for cleaner, smaller training data
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com
 Classifier: Development Status :: 4 - Beta
@@ -20,8 +20,10 @@ License-File: LICENSE
 Requires-Dist: numpy>=1.24.0
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: scikit-learn>=1.3.0
-Requires-Dist: torch>=2.0.0
+Provides-Extra: torch
+Requires-Dist: torch>=2.0.0; extra == "torch"
 Provides-Extra: clip
+Requires-Dist: torch>=2.0.0; extra == "clip"
 Requires-Dist: transformers>=4.30.0; extra == "clip"
 Requires-Dist: pillow>=9.0.0; extra == "clip"
 Dynamic: license-file

{datapruning-2.1.0 → datapruning-2.1.2}/datapruning/algorithms/base.py

@@ -47,7 +47,19 @@ class BaseOptimizer(ABC):
 
     def optimize(self, df: pd.DataFrame, target_col: str, keep_ratio: float = 0.5) -> OptimizationResult:
         import time
-        X, y = df.drop(columns=[target_col]), df[target_col]
+        import numpy as np
+        from sklearn.preprocessing import LabelEncoder
+
+        X, y = df.drop(columns=[target_col]), df[target_col].copy()
+
+        for col in X.columns:
+            if not np.issubdtype(X[col].dtype, np.number):
+                X[col] = LabelEncoder().fit_transform(X[col].astype(str))
+
+        if not np.issubdtype(y.dtype, np.number):
+            y = pd.Series(LabelEncoder().fit_transform(y.astype(str)), index=y.index)
+
+        X = X.fillna(0).astype(np.float64)
         minority_before = int(y.value_counts().min()) if y.nunique() > 1 else None
         t0 = time.time()
         self.fit(X, y)

{datapruning-2.1.0 → datapruning-2.1.2}/datapruning/algorithms/el2n_scorer.py

@@ -8,7 +8,7 @@ from datapruning.algorithms.registry import register
 @register
 class EL2NScorerOptimizer(BaseOptimizer):
     key = "el2n"
-    display_name = "Difficulty Scoring"
+    display_name = "Importance Scoring"
 
     def __init__(self):
         self._scores = None
@@ -43,14 +43,13 @@ class EL2NScorerOptimizer(BaseOptimizer):
         return minority_idx.append(kept_majority_idx)
 
     def why_selected(self) -> str:
-        return ("Difficulty Scoring uses feature-space distance analysis to identify "
-                "samples that are far from their class centroid. These informative "
-                "boundary samples help the model learn better decision regions. "
-                "Works as a reliable fallback for any dataset shape.")
+        return ("Measures how difficult each sample is for the model to classify by analyzing "
+                "feature-space distances. Keeps the most informative boundary samples that "
+                "help the model learn better decision regions. Works reliably on any dataset.")
 
     def limitations(self) -> list[str]:
         return [
-            "Uses feature norms as proxy — less precise than real model logits.",
+            "Uses distance-based approximation — slightly less precise than model-based methods.",
             "May select outliers on very noisy datasets.",
         ]
 

{datapruning-2.1.0 → datapruning-2.1.2}/datapruning/algorithms/fusion_pipeline.py

@@ -32,16 +32,14 @@ class FusionPipelineOptimizer(BaseOptimizer):
         return self._selected_indices
 
     def why_selected(self) -> str:
-        return ("Full Pipeline runs the complete ResearchFusionEngine: "
-                "multi-signal scoring (EL2N + Margin), normalization, "
-                "adaptive fusion, noise detection, difficulty classification, "
-                "redundancy removal, and diversity-aware selection. "
-                "Maximum quality for complex datasets.")
+        return ("Runs the most comprehensive analysis: combines multiple quality signals, "
+                "detects noise, classifies sample difficulty, removes redundancy, and "
+                "ensures diversity. Maximum quality for complex or messy datasets.")
 
     def limitations(self) -> list[str]:
         return [
-            "Slower than individual methods — runs multiple scoring steps.",
-            "May be overkill for simple, clean datasets.",
+            "Takes longer than simpler methods due to multiple analysis steps.",
+            "May be overkill for small, clean datasets.",
         ]
 
     @staticmethod

{datapruning-2.1.0 → datapruning-2.1.2}/datapruning/algorithms/kcenter_select.py

@@ -30,16 +30,14 @@ class KCenterSelectOptimizer(BaseOptimizer):
         return self._selected_indices
 
     def why_selected(self) -> str:
-        return ("Diversity Selection uses K-Center Greedy to pick the most "
-                "representative and diverse subset. It ensures broad coverage "
-                "of the feature space, avoiding redundancy. Ideal for "
-                "high-dimensional data or when you need maximum data variety.")
+        return ("Selects the most diverse and representative subset of your data by ensuring "
+                "broad coverage of the feature space. Eliminates redundant similar samples "
+                "while preserving maximum variety.")
 
     def limitations(self) -> list[str]:
         return [
-            "Does not consider class labels directly — relies on feature diversity.",
-            "May underperform on highly imbalanced datasets without minority preservation.",
-            "Slower than scoring-based methods on very large datasets.",
+            "Focuses on diversity rather than class labels directly.",
+            "May be less effective on highly imbalanced datasets.",
         ]
 
     @staticmethod

{datapruning-2.1.0 → datapruning-2.1.2}/datapruning/algorithms/margin_scorer.py

@@ -49,15 +49,14 @@ class MarginScorerOptimizer(BaseOptimizer):
         return minority_idx.append(kept_majority_idx)
 
     def why_selected(self) -> str:
-        return ("Margin Optimization uses logistic regression to measure how confidently "
-                "each sample is classified. It keeps the most informative majority-class "
-                "samples (high margin = clear signal) and preserves 100% of the minority class. "
-                "Best for general tabular data with clear class boundaries.")
+        return ("Analyzes how confidently each data point is classified, keeping the most "
+                "informative samples near decision boundaries. Automatically preserves all "
+                "minority-class examples to maintain balanced training data.")
 
     def limitations(self) -> list[str]:
         return [
-            "Requires fitting a quick logistic regression model.",
-            "May underperform on very high-dimensional data without feature selection.",
+            "Requires a quick model fitting step during analysis.",
+            "May be less effective on extremely high-dimensional datasets.",
         ]
 
     @staticmethod

{datapruning-2.1.0 → datapruning-2.1.2}/datapruning/algorithms/sliding_window.py

@@ -60,15 +60,14 @@ class SlidingWindowOptimizer(BaseOptimizer):
         return all_idx
 
     def why_selected(self) -> str:
-        return ("Noise Filtering drops both the easiest (redundant) and hardest "
-                "(potentially mislabeled) samples, keeping the informative middle. "
-                "This is the sliding window method from the EL2N/GraNd paper. "
-                "Best for datasets with suspected label noise.")
+        return ("Filters out both the easiest (redundant) and hardest (potentially mislabeled) "
+                "samples, keeping only the informative middle ground. Best for cleaning "
+                "datasets that may contain labeling errors or noisy data.")
 
     def limitations(self) -> list[str]:
         return [
-            "Requires fitting a logistic regression for scoring.",
-            "May discard genuinely hard but correct samples on complex datasets.",
+            "Requires a quick model fitting step during analysis.",
+            "May occasionally remove genuinely hard but correct samples.",
         ]
 
     @staticmethod

{datapruning-2.1.0 → datapruning-2.1.2}/datapruning/engine/strategy_display.py

@@ -2,23 +2,23 @@ STRATEGY_DISPLAY = {
     "margin": {
         "display_name": "Margin Optimization",
         "icon": "\U0001F3AF",
-        "description": "Uses logistic regression to measure classification confidence. Keeps the most informative samples near decision boundaries while preserving all minority-class examples.",
+        "description": "Analyzes classification confidence to keep the most informative samples near decision boundaries while preserving all minority-class examples.",
         "best_for": "General tabular data, binary and multi-class classification",
         "advantages": [
             "Best overall accuracy on benchmarks (+12% avg improvement)",
-            "Fast execution — single logistic regression fit",
+            "Fast execution -- single analysis step",
             "Preserves 100% of minority class",
         ],
         "tradeoffs": [
-            "Requires fitting a quick model",
-            "May underperform on extremely high-dimensional data",
+            "Requires a quick model fitting step",
+            "May be less effective on extremely high-dimensional data",
         ],
         "recommended_models": ["XGBoost", "LightGBM", "Random Forest", "CatBoost"],
     },
     "el2n": {
-        "display_name": "Difficulty Scoring",
+        "display_name": "Importance Scoring",
         "icon": "\U0001F4CA",
-        "description": "Analyzes feature-space distances to identify informative boundary samples. Works as a reliable fallback for any dataset shape.",
+        "description": "Measures how difficult each sample is to classify by analyzing feature distances. Keeps the most informative boundary samples.",
         "best_for": "Fallback method, any dataset type, quick analysis",
         "advantages": [
             "Works on any dataset without model training",
@@ -29,12 +29,12 @@ STRATEGY_DISPLAY = {
             "Less precise than model-based scoring",
             "May select outliers on very noisy datasets",
         ],
-        "recommended_models": ["Any model — general purpose"],
+        "recommended_models": ["Any model -- general purpose"],
     },
     "kcenter": {
         "display_name": "Diversity Selection",
         "icon": "\U0001F308",
-        "description": "Picks the most representative and diverse subset using K-Center Greedy. Ensures broad coverage of the feature space, avoiding redundancy.",
+        "description": "Picks the most representative and diverse subset, ensuring broad coverage of the feature space while avoiding redundancy.",
         "best_for": "High-dimensional data, datasets with redundancy, when variety matters",
         "advantages": [
             "Maximum feature space coverage",
@@ -50,34 +50,34 @@ STRATEGY_DISPLAY = {
     "sliding_window": {
         "display_name": "Noise Filtering",
         "icon": "\U0001F9FB",
-        "description": "Drops both the easiest (redundant) and hardest (potentially mislabeled) samples, keeping the informative middle. Based on the EL2N/GraNd research paper.",
+        "description": "Drops both the easiest (redundant) and hardest (potentially mislabeled) samples, keeping the informative middle ground.",
         "best_for": "Datasets with suspected label noise, corrupted data",
         "advantages": [
             "Removes both redundant and noisy samples",
-            "Research-backed method from top ML papers",
+            "Proven method for cleaning real-world data",
             "Good for cleaning real-world messy data",
         ],
         "tradeoffs": [
-            "Requires fitting a logistic regression",
-            "May discard genuinely hard but correct samples",
+            "Requires a quick model fitting step",
+            "May occasionally discard hard but correct samples",
         ],
         "recommended_models": ["XGBoost", "LightGBM", "Any robust model"],
     },
     "fusion": {
         "display_name": "Full Pipeline",
         "icon": "\u2728",
-        "description": "Runs the complete ResearchFusionEngine: multi-signal scoring, normalization, adaptive fusion, noise detection, difficulty classification, redundancy removal, and diversity-aware selection. Maximum quality.",
+        "description": "The most comprehensive analysis: combines multiple quality signals, detects noise, classifies importance, removes redundancy, and ensures diversity.",
         "best_for": "Complex datasets, when quality matters more than speed",
         "advantages": [
-            "Most comprehensive pruning pipeline",
-            "Combines multiple research-backed signals",
+            "Most comprehensive optimization pipeline",
+            "Combines multiple quality signals",
             "Handles noise, redundancy, and diversity",
         ],
         "tradeoffs": [
             "Slower than individual methods",
             "May be overkill for simple clean datasets",
         ],
-        "recommended_models": ["Any — maximizes data quality"],
+        "recommended_models": ["Any -- maximizes data quality"],
     },
 }
 

{datapruning-2.1.0 → datapruning-2.1.2}/datapruning/research_pruning/difficulty.py

@@ -32,6 +32,12 @@ def classify_difficulty(
     thresholds : dict — the score values at each boundary.
     """
     s = np.asarray(scores, dtype=np.float64)
+    score_range = s.max() - s.min()
+
+    if score_range < 1e-10:
+        labels = np.full(len(s), "medium", dtype="U6")
+        return labels, {"easy_below": float(s.min()), "hard_above": float(s.max())}
+
     easy_val = np.percentile(s, easy_threshold)
     hard_val = np.percentile(s, hard_threshold)
 

{datapruning-2.1.0 → datapruning-2.1.2}/datapruning/research_pruning/pipeline.py

@@ -209,12 +209,14 @@ class ResearchFusionEngine:
         adjusted_scores[noise_mask] -= np.abs(adjusted_scores[noise_mask].max()) + 1
 
         if selection_method == "sliding_window":
+            budget = max(1, int(n * keep_ratio))
             selected = sliding_window_select(
                 adjusted_scores,
                 low_percentile=10.0,
                 high_percentile=90.0,
                 labels=y,
                 preserve_minority=minority_preserve,
+                budget=budget,
             )
         else:
             budget = max(1, int(n * keep_ratio))

{datapruning-2.1.0 → datapruning-2.1.2}/datapruning/research_pruning/selection.py

@@ -88,40 +88,45 @@ def sliding_window_select(
     high_percentile: float = 90.0,
     labels: np.ndarray | None = None,
     preserve_minority: bool = True,
+    budget: int | None = None,
 ) -> np.ndarray:
-    """Sliding window selection (from Paul et al., 2021).
+    """Sliding window selection — keep samples within a score range.
 
-    Keep samples within a score range, excluding both easiest and hardest.
-    This is the method described in the EL2N/GraNd paper for balancing
-    difficulty and coverage.
+    Excludes both the easiest (redundant) and hardest (potentially noisy)
+    samples, keeping the most informative middle band.
 
     Parameters
     ----------
-    scores : (N,) array — difficulty scores.
-    low_percentile : float — lower bound (drop easier samples).
+    scores : (N,) array — importance scores.
+    low_percentile : float — lower bound (drop easiest samples).
     high_percentile : float — upper bound (drop hardest/noisiest samples).
+    labels : (N,) array — class labels (for balance enforcement).
+    preserve_minority : bool — ensure minority classes aren't wiped out.
+    budget : int, optional — if set, cap output to this many samples.
     """
     s = np.asarray(scores, dtype=np.float64)
+    n = len(s)
     low_val = np.percentile(s, low_percentile)
     high_val = np.percentile(s, high_percentile)
 
     in_window = (s >= low_val) & (s <= high_val)
     window_indices = np.where(in_window)[0]
 
-    if labels is None or not preserve_minority:
-        return window_indices
-
-    # ensure minority classes are represented in the window
-    labels = np.asarray(labels, dtype=np.int64)
-    classes = np.unique(labels)
-    for cls in classes:
-        cls_in_window = window_indices[labels[window_indices] == cls]
-        if len(cls_in_window) == 0:
-            # no samples of this class in window → add top-scored ones
-            cls_mask = labels == cls
-            cls_scores = s.copy()
-            cls_scores[~cls_mask] = -np.inf
-            top_cls = np.argsort(cls_scores)[-1:]
-            window_indices = np.concatenate([window_indices, top_cls])
+    if labels is not None and preserve_minority:
+        labels = np.asarray(labels, dtype=np.int64)
+        classes = np.unique(labels)
+        for cls in classes:
+            cls_in_window = window_indices[labels[window_indices] == cls]
+            if len(cls_in_window) == 0:
+                cls_mask = labels == cls
+                cls_scores = s.copy()
+                cls_scores[~cls_mask] = -np.inf
+                top_cls = np.argsort(cls_scores)[-1:]
+                window_indices = np.concatenate([window_indices, top_cls])
+
+    if budget is not None and len(window_indices) > budget:
+        cls_in = window_indices
+        cls_sub = cls_in[np.argsort(s[cls_in])[-budget:][::-1]]
+        window_indices = cls_sub
 
     return window_indices

{datapruning-2.1.0 → datapruning-2.1.2}/datapruning/sdk.py

@@ -35,7 +35,7 @@ class DatasetEngine:
         """Create a DatasetEngine from a folder of images.
 
         Encodes all images with CLIP into feature vectors, then wraps
-        them in a DataFrame so the existing pruning algorithms work
+        them in a DataFrame so the existing optimization algorithms work
         unchanged.
 
         Parameters

{datapruning-2.1.0 → datapruning-2.1.2/datapruning.egg-info}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 2.1.0
-Summary: Intelligent data pruning for ML datasets
+Version: 2.1.2
+Summary: Intelligent dataset optimization for cleaner, smaller training data
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com
 Classifier: Development Status :: 4 - Beta
@@ -20,8 +20,10 @@ License-File: LICENSE
 Requires-Dist: numpy>=1.24.0
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: scikit-learn>=1.3.0
-Requires-Dist: torch>=2.0.0
+Provides-Extra: torch
+Requires-Dist: torch>=2.0.0; extra == "torch"
 Provides-Extra: clip
+Requires-Dist: torch>=2.0.0; extra == "clip"
 Requires-Dist: transformers>=4.30.0; extra == "clip"
 Requires-Dist: pillow>=9.0.0; extra == "clip"
 Dynamic: license-file

{datapruning-2.1.0 → datapruning-2.1.2}/datapruning.egg-info/requires.txt

@@ -1,8 +1,11 @@
 numpy>=1.24.0
 pandas>=2.0.0
 scikit-learn>=1.3.0
-torch>=2.0.0
 
 [clip]
+torch>=2.0.0
 transformers>=4.30.0
 pillow>=9.0.0
+
+[torch]
+torch>=2.0.0

{datapruning-2.1.0 → datapruning-2.1.2}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datapruning"
-version = "2.1.0"
-description = "Intelligent data pruning for ML datasets"
+version = "2.1.2"
+description = "Intelligent dataset optimization for cleaner, smaller training data"
 readme = "README.md"
 requires-python = ">=3.10"
 license = {text = "Proprietary"}
@@ -25,11 +25,14 @@ dependencies = [
     "numpy>=1.24.0",
     "pandas>=2.0.0",
     "scikit-learn>=1.3.0",
-    "torch>=2.0.0",
 ]
 
 [project.optional-dependencies]
+torch = [
+    "torch>=2.0.0",
+]
 clip = [
+    "torch>=2.0.0",
     "transformers>=4.30.0",
     "pillow>=9.0.0",
 ]