datapruning 1.0.10__tar.gz → 2.1.0__tar.gz

{datapruning-1.0.10/datapruning.egg-info → datapruning-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 1.0.10
+Version: 2.1.0
 Summary: Intelligent data pruning for ML datasets
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com
@@ -21,6 +21,9 @@ Requires-Dist: numpy>=1.24.0
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: scikit-learn>=1.3.0
 Requires-Dist: torch>=2.0.0
+Provides-Extra: clip
+Requires-Dist: transformers>=4.30.0; extra == "clip"
+Requires-Dist: pillow>=9.0.0; extra == "clip"
 Dynamic: license-file
 
 # DataPruning

datapruning-2.1.0/datapruning/algorithms/__init__.py

@@ -0,0 +1,7 @@
+from datapruning.algorithms import margin_scorer
+from datapruning.algorithms import el2n_scorer
+from datapruning.algorithms import kcenter_select
+from datapruning.algorithms import sliding_window
+from datapruning.algorithms import fusion_pipeline
+
+from datapruning.algorithms.registry import registry

datapruning-2.1.0/datapruning/algorithms/el2n_scorer.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+import numpy as np
+import pandas as pd
+from datapruning.algorithms.base import BaseOptimizer
+from datapruning.algorithms.registry import register
+
+
+@register
+class EL2NScorerOptimizer(BaseOptimizer):
+    key = "el2n"
+    display_name = "Difficulty Scoring"
+
+    def __init__(self):
+        self._scores = None
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "EL2NScorerOptimizer":
+        X_arr = X.values.astype(np.float64)
+        y_arr = y.values
+        feat_norms = np.linalg.norm(X_arr, axis=1)
+        feat_norms_normed = (feat_norms - feat_norms.min()) / max(feat_norms.max() - feat_norms.min(), 1e-8)
+        classes = np.unique(y_arr)
+        centroid_scores = np.zeros(len(y_arr), dtype=np.float64)
+        for cls in classes:
+            cls_mask = y_arr == cls
+            centroid = X_arr[cls_mask].mean(axis=0)
+            dists = np.linalg.norm(X_arr - centroid, axis=1)
+            dists_normed = (dists - dists.min()) / max(dists.max() - dists.min(), 1e-8)
+            centroid_scores = np.maximum(centroid_scores, dists_normed)
+        combined = 0.5 * feat_norms_normed + 0.5 * centroid_scores
+        self._scores = pd.Series(combined, index=y.index)
+        return self
+
+    def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
+        if self._scores is None:
+            self.fit(X, y)
+        minority_class = y.value_counts().idxmin()
+        minority_idx = y[y == minority_class].index
+        majority_idx = y[y != minority_class].index
+        n_total_target = int(len(y) * keep_ratio)
+        n_majority_target = max(n_total_target - len(minority_idx), 0)
+        majority_scores = self._scores.loc[majority_idx].sort_values(ascending=False)
+        kept_majority_idx = majority_scores.index[:n_majority_target]
+        return minority_idx.append(kept_majority_idx)
+
+    def why_selected(self) -> str:
+        return ("Difficulty Scoring uses feature-space distance analysis to identify "
+                "samples that are far from their class centroid. These informative "
+                "boundary samples help the model learn better decision regions. "
+                "Works as a reliable fallback for any dataset shape.")
+
+    def limitations(self) -> list[str]:
+        return [
+            "Uses feature norms as proxy — less precise than real model logits.",
+            "May select outliers on very noisy datasets.",
+        ]
+
+    @staticmethod
+    def fit_score(analysis: dict) -> float:
+        scan = analysis.get("scan", analysis)
+        task_type = scan.get("task_type", "unknown")
+        n_rows = scan.get("rows", 0)
+
+        if task_type == "regression":
+            return 0.0
+        if n_rows < 200:
+            return 0.1
+        return 0.5

datapruning-2.1.0/datapruning/algorithms/fusion_pipeline.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+import numpy as np
+import pandas as pd
+from datapruning.research_pruning.pipeline import ResearchFusionEngine
+from datapruning.algorithms.base import BaseOptimizer
+from datapruning.algorithms.registry import register
+
+
+@register
+class FusionPipelineOptimizer(BaseOptimizer):
+    key = "fusion"
+    display_name = "Full Pipeline"
+
+    def __init__(self):
+        self._selected_indices = None
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "FusionPipelineOptimizer":
+        self._selected_indices = None
+        return self
+
+    def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
+        X_arr = X.values.astype(np.float64)
+        y_arr = y.values.astype(np.int64)
+        engine = ResearchFusionEngine(config={"keep_ratio": keep_ratio})
+        pruned_X, pruned_y, report = engine.run(
+            X_arr, y_arr,
+            keep_ratio=keep_ratio,
+            auto_strategy=True,
+        )
+        selected = report.get("selected_indices", [])
+        self._selected_indices = X.index[selected]
+        return self._selected_indices
+
+    def why_selected(self) -> str:
+        return ("Full Pipeline runs the complete ResearchFusionEngine: "
+                "multi-signal scoring (EL2N + Margin), normalization, "
+                "adaptive fusion, noise detection, difficulty classification, "
+                "redundancy removal, and diversity-aware selection. "
+                "Maximum quality for complex datasets.")
+
+    def limitations(self) -> list[str]:
+        return [
+            "Slower than individual methods — runs multiple scoring steps.",
+            "May be overkill for simple, clean datasets.",
+        ]
+
+    @staticmethod
+    def fit_score(analysis: dict) -> float:
+        scan = analysis.get("scan", analysis)
+        task_type = scan.get("task_type", "unknown")
+        n_rows = scan.get("rows", 0)
+        n_cols = scan.get("columns", 0)
+        duplicate_pct = scan.get("duplicate_pct", 0)
+        imbalance = scan.get("imbalance_ratio", 1.0)
+
+        if task_type == "regression":
+            return 0.0
+        if n_rows < 1000:
+            return 0.2
+        score = 0.5
+        if n_rows > 10000:
+            score += 0.15
+        if n_cols > 30:
+            score += 0.1
+        if duplicate_pct > 3:
+            score += 0.1
+        if imbalance > 3:
+            score += 0.1
+        return min(score, 0.9)

datapruning-2.1.0/datapruning/algorithms/kcenter_select.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+import numpy as np
+import pandas as pd
+from sklearn.preprocessing import StandardScaler
+from datapruning.research_pruning.selectors.kcenter import kcenter_greedy
+from datapruning.algorithms.base import BaseOptimizer
+from datapruning.algorithms.registry import register
+
+
+@register
+class KCenterSelectOptimizer(BaseOptimizer):
+    key = "kcenter"
+    display_name = "Diversity Selection"
+
+    def __init__(self):
+        self._selected_indices = None
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "KCenterSelectOptimizer":
+        self._selected_indices = None
+        return self
+
+    def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
+        X_arr = X.values.astype(np.float64)
+        n = len(X_arr)
+        budget = max(1, int(n * keep_ratio))
+        scaler = StandardScaler()
+        X_scaled = scaler.fit_transform(X_arr)
+        selected = kcenter_greedy(X_scaled, budget=min(budget, n), random_seed=42)
+        self._selected_indices = X.index[selected]
+        return self._selected_indices
+
+    def why_selected(self) -> str:
+        return ("Diversity Selection uses K-Center Greedy to pick the most "
+                "representative and diverse subset. It ensures broad coverage "
+                "of the feature space, avoiding redundancy. Ideal for "
+                "high-dimensional data or when you need maximum data variety.")
+
+    def limitations(self) -> list[str]:
+        return [
+            "Does not consider class labels directly — relies on feature diversity.",
+            "May underperform on highly imbalanced datasets without minority preservation.",
+            "Slower than scoring-based methods on very large datasets.",
+        ]
+
+    @staticmethod
+    def fit_score(analysis: dict) -> float:
+        scan = analysis.get("scan", analysis)
+        task_type = scan.get("task_type", "unknown")
+        n_rows = scan.get("rows", 0)
+        n_cols = scan.get("columns", 0)
+
+        if task_type == "regression":
+            return 0.2
+        if n_cols > n_rows:
+            return 0.8
+        if n_cols > 50:
+            return 0.65
+        return 0.4

datapruning-2.1.0/datapruning/algorithms/margin_scorer.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+import numpy as np
+import pandas as pd
+from sklearn.linear_model import LogisticRegression
+from datapruning.algorithms.base import BaseOptimizer
+from datapruning.algorithms.registry import register
+
+
+@register
+class MarginScorerOptimizer(BaseOptimizer):
+    key = "margin"
+    display_name = "Margin Optimization"
+
+    def __init__(self):
+        self._scores = None
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "MarginScorerOptimizer":
+        X_arr, y_arr = X.values.astype(np.float64), y.values
+        classes = np.unique(y_arr)
+        if len(classes) < 2:
+            self._scores = pd.Series(np.zeros(len(y_arr)), index=y.index)
+            return self
+        try:
+            lr = LogisticRegression(max_iter=1000, random_state=42)
+            lr.fit(X_arr, y_arr)
+            probs = lr.predict_proba(X_arr)
+            class_to_col = {c: i for i, c in enumerate(lr.classes_)}
+            true_probs = np.array([probs[i, class_to_col[y_arr[i]]] for i in range(len(y_arr))])
+            self._scores = pd.Series(true_probs, index=y.index)
+        except Exception:
+            feat_norms = np.linalg.norm(X_arr, axis=1)
+            self._scores = pd.Series(feat_norms, index=y.index)
+        return self
+
+    def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
+        if self._scores is None:
+            self.fit(X, y)
+        classes = y.unique()
+        if len(classes) < 2:
+            n_target = int(len(y) * keep_ratio)
+            return y.nlargest(n_target).index
+        minority_class = y.value_counts().idxmin()
+        minority_idx = y[y == minority_class].index
+        majority_idx = y[y != minority_class].index
+        n_total_target = int(len(y) * keep_ratio)
+        n_majority_target = max(n_total_target - len(minority_idx), 0)
+        majority_scores = self._scores.loc[majority_idx].sort_values(ascending=False)
+        kept_majority_idx = majority_scores.index[:n_majority_target]
+        return minority_idx.append(kept_majority_idx)
+
+    def why_selected(self) -> str:
+        return ("Margin Optimization uses logistic regression to measure how confidently "
+                "each sample is classified. It keeps the most informative majority-class "
+                "samples (high margin = clear signal) and preserves 100% of the minority class. "
+                "Best for general tabular data with clear class boundaries.")
+
+    def limitations(self) -> list[str]:
+        return [
+            "Requires fitting a quick logistic regression model.",
+            "May underperform on very high-dimensional data without feature selection.",
+        ]
+
+    @staticmethod
+    def fit_score(analysis: dict) -> float:
+        scan = analysis.get("scan", analysis)
+        task_type = scan.get("task_type", "unknown")
+        n_rows = scan.get("rows", 0)
+        n_cols = scan.get("columns", 0)
+
+        if task_type == "regression":
+            return 0.0
+        if n_rows < 500:
+            return 0.1
+        if n_rows > 200_000:
+            return 0.6
+        if n_cols > 100:
+            return 0.7
+        return 0.85

datapruning-2.1.0/datapruning/algorithms/sliding_window.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+import numpy as np
+import pandas as pd
+from sklearn.linear_model import LogisticRegression
+from datapruning.research_pruning.selection import sliding_window_select
+from datapruning.algorithms.base import BaseOptimizer
+from datapruning.algorithms.registry import register
+
+
+@register
+class SlidingWindowOptimizer(BaseOptimizer):
+    key = "sliding_window"
+    display_name = "Noise Filtering"
+
+    def __init__(self):
+        self._scores = None
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "SlidingWindowOptimizer":
+        X_arr, y_arr = X.values.astype(np.float64), y.values
+        classes = np.unique(y_arr)
+        if len(classes) < 2:
+            self._scores = np.zeros(len(y_arr))
+            return self
+        try:
+            lr = LogisticRegression(max_iter=1000, random_state=42)
+            lr.fit(X_arr, y_arr)
+            probs = lr.predict_proba(X_arr)
+            class_to_col = {c: i for i, c in enumerate(lr.classes_)}
+            true_probs = np.array([probs[i, class_to_col[y_arr[i]]] for i in range(len(y_arr))])
+            self._scores = true_probs
+        except Exception:
+            self._scores = np.linalg.norm(X_arr, axis=1)
+        return self
+
+    def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
+        if self._scores is None:
+            self.fit(X, y)
+        y_arr = y.values
+        classes = np.unique(y_arr)
+        low_pct = max(0, (1 - keep_ratio) * 100 / 2)
+        high_pct = min(100, 100 - low_pct)
+        if len(classes) < 2:
+            n_target = int(len(y) * keep_ratio)
+            return y.nlargest(n_target).index
+        selected = sliding_window_select(
+            self._scores,
+            low_percentile=low_pct,
+            high_percentile=high_pct,
+            labels=y_arr,
+            preserve_minority=True,
+        )
+        minority_class = y.value_counts().idxmin()
+        minority_idx = y[y == minority_class].index
+        selected_idx = X.index[selected]
+        all_idx = selected_idx.union(minority_idx).unique()
+        n_target = int(len(y) * keep_ratio)
+        if len(all_idx) > n_target:
+            scores_at_idx = pd.Series(self._scores, index=X.index).loc[all_idx]
+            all_idx = scores_at_idx.sort_values(ascending=False).index[:n_target]
+        return all_idx
+
+    def why_selected(self) -> str:
+        return ("Noise Filtering drops both the easiest (redundant) and hardest "
+                "(potentially mislabeled) samples, keeping the informative middle. "
+                "This is the sliding window method from the EL2N/GraNd paper. "
+                "Best for datasets with suspected label noise.")
+
+    def limitations(self) -> list[str]:
+        return [
+            "Requires fitting a logistic regression for scoring.",
+            "May discard genuinely hard but correct samples on complex datasets.",
+        ]
+
+    @staticmethod
+    def fit_score(analysis: dict) -> float:
+        scan = analysis.get("scan", analysis)
+        task_type = scan.get("task_type", "unknown")
+        n_rows = scan.get("rows", 0)
+        duplicate_pct = scan.get("duplicate_pct", 0)
+
+        if task_type == "regression":
+            return 0.0
+        if n_rows < 500:
+            return 0.1
+        if duplicate_pct > 5:
+            return 0.75
+        return 0.6

datapruning-2.1.0/datapruning/engine/strategy_display.py

@@ -0,0 +1,118 @@
+STRATEGY_DISPLAY = {
+    "margin": {
+        "display_name": "Margin Optimization",
+        "icon": "\U0001F3AF",
+        "description": "Uses logistic regression to measure classification confidence. Keeps the most informative samples near decision boundaries while preserving all minority-class examples.",
+        "best_for": "General tabular data, binary and multi-class classification",
+        "advantages": [
+            "Best overall accuracy on benchmarks (+12% avg improvement)",
+            "Fast execution — single logistic regression fit",
+            "Preserves 100% of minority class",
+        ],
+        "tradeoffs": [
+            "Requires fitting a quick model",
+            "May underperform on extremely high-dimensional data",
+        ],
+        "recommended_models": ["XGBoost", "LightGBM", "Random Forest", "CatBoost"],
+    },
+    "el2n": {
+        "display_name": "Difficulty Scoring",
+        "icon": "\U0001F4CA",
+        "description": "Analyzes feature-space distances to identify informative boundary samples. Works as a reliable fallback for any dataset shape.",
+        "best_for": "Fallback method, any dataset type, quick analysis",
+        "advantages": [
+            "Works on any dataset without model training",
+            "Reliable baseline performance",
+            "Very fast execution",
+        ],
+        "tradeoffs": [
+            "Less precise than model-based scoring",
+            "May select outliers on very noisy datasets",
+        ],
+        "recommended_models": ["Any model — general purpose"],
+    },
+    "kcenter": {
+        "display_name": "Diversity Selection",
+        "icon": "\U0001F308",
+        "description": "Picks the most representative and diverse subset using K-Center Greedy. Ensures broad coverage of the feature space, avoiding redundancy.",
+        "best_for": "High-dimensional data, datasets with redundancy, when variety matters",
+        "advantages": [
+            "Maximum feature space coverage",
+            "Eliminates redundant samples",
+            "Great for high-dimensional data",
+        ],
+        "tradeoffs": [
+            "Does not consider class labels directly",
+            "Slower on very large datasets",
+        ],
+        "recommended_models": ["LightGBM", "Random Forest", "Any ensemble"],
+    },
+    "sliding_window": {
+        "display_name": "Noise Filtering",
+        "icon": "\U0001F9FB",
+        "description": "Drops both the easiest (redundant) and hardest (potentially mislabeled) samples, keeping the informative middle. Based on the EL2N/GraNd research paper.",
+        "best_for": "Datasets with suspected label noise, corrupted data",
+        "advantages": [
+            "Removes both redundant and noisy samples",
+            "Research-backed method from top ML papers",
+            "Good for cleaning real-world messy data",
+        ],
+        "tradeoffs": [
+            "Requires fitting a logistic regression",
+            "May discard genuinely hard but correct samples",
+        ],
+        "recommended_models": ["XGBoost", "LightGBM", "Any robust model"],
+    },
+    "fusion": {
+        "display_name": "Full Pipeline",
+        "icon": "\u2728",
+        "description": "Runs the complete ResearchFusionEngine: multi-signal scoring, normalization, adaptive fusion, noise detection, difficulty classification, redundancy removal, and diversity-aware selection. Maximum quality.",
+        "best_for": "Complex datasets, when quality matters more than speed",
+        "advantages": [
+            "Most comprehensive pruning pipeline",
+            "Combines multiple research-backed signals",
+            "Handles noise, redundancy, and diversity",
+        ],
+        "tradeoffs": [
+            "Slower than individual methods",
+            "May be overkill for simple clean datasets",
+        ],
+        "recommended_models": ["Any — maximizes data quality"],
+    },
+}
+
+
+def get_strategy_display(key: str) -> dict:
+    return dict(STRATEGY_DISPLAY.get(key, {
+        "display_name": key.replace("_", " ").title(),
+        "icon": "\U0001F527",
+        "description": "",
+        "best_for": "",
+        "advantages": [],
+        "tradeoffs": [],
+        "recommended_models": [],
+    }))
+
+
+def enrich_algorithm_info(algo: dict) -> dict:
+    display = get_strategy_display(algo.get("key", ""))
+    return {**algo, **display}
+
+
+def enrich_recommendation(rec: dict) -> dict:
+    key = rec.get("algorithm_key", "")
+    display = get_strategy_display(key)
+    enriched = {
+        "friendly_name": display["display_name"],
+        "icon": display["icon"],
+        "description": display["description"],
+        "best_for": display["best_for"],
+        "advantages": display["advantages"],
+        "tradeoffs": display["tradeoffs"],
+        "recommended_models": display["recommended_models"],
+    }
+    return {**rec, **enriched}
+
+
+def enrich_alternatives(alternatives: list) -> list:
+    return [enrich_algorithm_info(alt) for alt in alternatives]

datapruning-2.1.0/datapruning/multimodal/__init__.py

@@ -0,0 +1 @@
+"""Multimodal pruning support — encode images and text with CLIP, then prune."""