datapruning 1.0.6__tar.gz → 1.0.7__tar.gz

{datapruning-1.0.6/datapruning.egg-info → datapruning-1.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 1.0.6
+Version: 1.0.7
 Summary: Intelligent data pruning for ML datasets
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com
@@ -16,9 +16,12 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: numpy>=1.24.0
 Requires-Dist: pandas>=2.0.0
+Requires-Dist: scikit-learn>=1.3.0
 Requires-Dist: torch>=2.0.0
+Dynamic: license-file
 
 # DataPruning
 

{datapruning-1.0.6 → datapruning-1.0.7}/datapruning/__init__.py

@@ -3,15 +3,29 @@ import numpy as np
 import pandas as pd
 
 from . import pipeline
-from ._core import analyze_data as _analyze_data
+from .sdk import DatasetEngine
 
 logger = logging.getLogger(__name__)
 
 MIN_ROWS = 1000
 MAX_ROWS = 300_000
 
+try:
+    from ._core import analyze_data as _analyze_data
+    HAS_CORE = True
+except ImportError:
+    _analyze_data = None
+    HAS_CORE = False
+    logger.warning("Compiled _core module not available — legacy prune_dataframe/Pruner disabled")
+
 
 def prune_dataframe(df, target_col=None, keep_ratio=0.5, num_steps=400, verbose=False):
+    if not HAS_CORE:
+        raise RuntimeError(
+            "prune_dataframe requires the compiled _core module, "
+            "which is not available on this platform. "
+            "Use DatasetEngine instead (pure Python, all platforms)."
+        )
     if verbose:
         logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
 
@@ -53,6 +67,12 @@ class Pruner:
     def __init__(self, df):
         if not isinstance(df, pd.DataFrame):
             raise TypeError("Expected a pandas DataFrame")
+        if not HAS_CORE:
+            raise RuntimeError(
+                "Pruner requires the compiled _core module, "
+                "which is not available on this platform. "
+                "Use DatasetEngine instead (pure Python, all platforms)."
+            )
         self._df = df
 
     def prune(self, target_col=None, keep_ratio=0.5, num_steps=400, verbose=False):

datapruning-1.0.7/datapruning/algorithms/__init__.py

@@ -0,0 +1,7 @@
+from datapruning.algorithms import random_undersampler
+from datapruning.algorithms import cluster_centroids
+from datapruning.algorithms import instance_hardness
+from datapruning.algorithms import duplicate_aware
+from datapruning.algorithms import density_pruner
+
+from datapruning.algorithms.registry import registry

datapruning-1.0.7/datapruning/algorithms/base.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Optional
+import pandas as pd
+
+
+@dataclass
+class OptimizationResult:
+    optimized_df: pd.DataFrame
+    rows_before: int
+    rows_after: int
+    cols_before: int
+    cols_after: int
+    minority_count_before: Optional[int]
+    minority_count_after: Optional[int]
+    runtime_seconds: float
+
+
+@dataclass
+class Explanation:
+    algorithm: str
+    why_selected: str
+    rows_removed: int
+    reduction_pct: float
+    minority_preserved_pct: Optional[float]
+    estimated_training_gain_pct: float
+    memory_saved_pct: float
+    limitations: list[str] = field(default_factory=list)
+
+
+class BaseOptimizer(ABC):
+    key: str = ""
+    display_name: str = ""
+
+    @abstractmethod
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "BaseOptimizer":
+        raise NotImplementedError
+
+    @abstractmethod
+    def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
+        raise NotImplementedError
+
+    def transform(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float):
+        keep_idx = self._select_rows(X, y, keep_ratio)
+        return X.loc[keep_idx], y.loc[keep_idx]
+
+    def optimize(self, df: pd.DataFrame, target_col: str, keep_ratio: float = 0.5) -> OptimizationResult:
+        import time
+        X, y = df.drop(columns=[target_col]), df[target_col]
+        minority_before = int(y.value_counts().min()) if y.nunique() > 1 else None
+        t0 = time.time()
+        self.fit(X, y)
+        X_res, y_res = self.transform(X, y, keep_ratio)
+        runtime = time.time() - t0
+        minority_after = int(y_res.value_counts().min()) if y_res.nunique() > 1 else None
+        optimized_df = X_res.copy()
+        optimized_df[target_col] = y_res.values
+        return OptimizationResult(
+            optimized_df=optimized_df,
+            rows_before=len(df),
+            rows_after=len(optimized_df),
+            cols_before=df.shape[1],
+            cols_after=optimized_df.shape[1],
+            minority_count_before=minority_before,
+            minority_count_after=minority_after,
+            runtime_seconds=runtime,
+        )
+
+    def explain(self, result: OptimizationResult) -> Explanation:
+        rows_removed = result.rows_before - result.rows_after
+        reduction_pct = round(rows_removed / result.rows_before * 100, 1) if result.rows_before else 0.0
+        minority_preserved_pct = None
+        if result.minority_count_before:
+            minority_preserved_pct = round(
+                (result.minority_count_after or 0) / result.minority_count_before * 100, 1
+            )
+        return Explanation(
+            algorithm=self.display_name,
+            why_selected=self.why_selected(),
+            rows_removed=rows_removed,
+            reduction_pct=reduction_pct,
+            minority_preserved_pct=minority_preserved_pct,
+            estimated_training_gain_pct=reduction_pct,
+            memory_saved_pct=reduction_pct,
+            limitations=self.limitations(),
+        )
+
+    @staticmethod
+    @abstractmethod
+    def fit_score(analysis: dict) -> float:
+        raise NotImplementedError
+
+    def why_selected(self) -> str:
+        return f"{self.display_name} was selected as the best fit for this dataset."
+
+    def limitations(self) -> list[str]:
+        return []

datapruning-1.0.7/datapruning/algorithms/cluster_centroids.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+import numpy as np
+import pandas as pd
+from sklearn.cluster import KMeans
+from datapruning.algorithms.base import BaseOptimizer
+from datapruning.algorithms.registry import register
+
+
+@register
+class ClusterCentroidsOptimizer(BaseOptimizer):
+    key = "cluster"
+    display_name = "Cluster Centroids"
+
+    def __init__(self, random_state: int = 42):
+        self.random_state = random_state
+        self._centroid_rows = None
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "ClusterCentroidsOptimizer":
+        return self
+
+    def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
+        minority_class = y.value_counts().idxmin()
+        minority_idx = y[y == minority_class].index
+        majority_idx = y[y != minority_class].index
+        n_total_target = int(len(y) * keep_ratio)
+        n_majority_target = max(n_total_target - len(minority_idx), 0)
+        n_majority_target = min(n_majority_target, len(majority_idx))
+        if n_majority_target == 0:
+            return minority_idx
+        numeric_X = X.loc[majority_idx].select_dtypes(include=[np.number]).fillna(0)
+        k = max(min(n_majority_target, len(majority_idx), 3000), 1)
+        km = KMeans(n_clusters=k, random_state=self.random_state, n_init=3)
+        km.fit(numeric_X)
+        labels = km.labels_
+        per_cluster_quota = max(n_majority_target // k, 1)
+        kept_rows = []
+        numeric_vals = numeric_X.values
+        for cluster_id in range(k):
+            cluster_positions = np.where(labels == cluster_id)[0]
+            if len(cluster_positions) == 0:
+                continue
+            centroid = km.cluster_centers_[cluster_id]
+            dists = np.linalg.norm(numeric_vals[cluster_positions] - centroid, axis=1)
+            order = np.argsort(dists)
+            chosen_positions = cluster_positions[order[:per_cluster_quota]]
+            kept_rows.extend(numeric_X.index[chosen_positions].tolist())
+        keep_idx = pd.Index(kept_rows).append(minority_idx)
+        return keep_idx
+
+    def why_selected(self) -> str:
+        return ("Large balanced-to-moderately-imbalanced dataset where structural "
+                "(density-based) reduction outperforms blind random sampling.")
+
+    def limitations(self) -> list[str]:
+        return ["O(n*k) clustering cost — slow on very large datasets (100K+ rows).",
+                "Less effective than Instance Hardness Threshold on severe imbalance."]
+
+    @staticmethod
+    def fit_score(analysis: dict) -> float:
+        scan = analysis.get("scan", analysis)
+        task_type = scan.get("task_type", "unknown")
+        imbalance_ratio = scan.get("imbalance_ratio", 1.0)
+        n_rows = scan.get("rows", 0)
+
+        if n_rows < 1000 or task_type == "regression":
+            return 0.0
+        if n_rows > 200_000:
+            return 0.1
+        if 3 <= imbalance_ratio < 10:
+            return 0.6
+        return 0.3

datapruning-1.0.7/datapruning/algorithms/density_pruner.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+import numpy as np
+import pandas as pd
+from datapruning.algorithms.base import BaseOptimizer
+from datapruning.algorithms.registry import register
+
+
+@register
+class DensityPrunerOptimizer(BaseOptimizer):
+    key = "density"
+    display_name = "Density-Based Pruner (Legacy)"
+
+    def __init__(self, random_state: int = 42):
+        self.random_state = random_state
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "DensityPrunerOptimizer":
+        return self
+
+    def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
+        try:
+            from datapruning._core import analyze_data as core_prune
+            n = len(y)
+            n_keep = int(n * keep_ratio)
+            X_arr = np.ascontiguousarray(X.values.astype(np.float32))
+            result = core_prune(X_arr, y.values.copy(), 200, keep_ratio)
+            if result is not None and len(result) > 0:
+                kept = result[:n_keep]
+                return pd.Index(kept)
+        except Exception:
+            pass
+        rng = np.random.RandomState(self.random_state)
+        minority_class = y.value_counts().idxmin()
+        minority_idx = y[y == minority_class].index
+        majority_idx = y[y != minority_class].index
+        n_total_target = int(len(y) * keep_ratio)
+        n_majority_target = max(n_total_target - len(minority_idx), 0)
+        chosen_majority = rng.choice(majority_idx.to_numpy(),
+                                      size=min(n_majority_target, len(majority_idx)),
+                                      replace=False)
+        return minority_idx.append(pd.Index(chosen_majority))
+
+    def why_selected(self) -> str:
+        return "Dataset has structural density patterns that the legacy pruner can exploit."
+
+    def limitations(self) -> list[str]:
+        return ["Legacy algorithm — may not perform as well as newer methods.",
+                "Falls back to random undersampling if the compiled core is unavailable."]
+
+    @staticmethod
+    def fit_score(analysis: dict) -> float:
+        scan = analysis.get("scan", analysis)
+        n_rows = scan.get("rows", 0)
+        if n_rows < 1000:
+            return 0.0
+        task_type = scan.get("task_type", "unknown")
+        if task_type == "regression":
+            return 0.0
+        imbalance_ratio = scan.get("imbalance_ratio", 1.0)
+        if 1.5 <= imbalance_ratio < 5:
+            return 0.65
+        return 0.35

datapruning-1.0.7/datapruning/algorithms/duplicate_aware.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+import numpy as np
+import pandas as pd
+from datapruning.algorithms.base import BaseOptimizer
+from datapruning.algorithms.registry import register
+
+
+@register
+class DuplicateAwareOptimizer(BaseOptimizer):
+    key = "duplicate_aware"
+    display_name = "Duplicate-Aware Reduction"
+
+    def __init__(self, random_state: int = 42):
+        self.random_state = random_state
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "DuplicateAwareOptimizer":
+        return self
+
+    def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
+        rng = np.random.RandomState(self.random_state)
+        dedup_mask = ~X.duplicated()
+        deduped_idx = X.index[dedup_mask]
+        n_target = int(len(y) * keep_ratio)
+        if len(deduped_idx) <= n_target:
+            return deduped_idx
+        minority_class = y.value_counts().idxmin()
+        minority_idx = deduped_idx.intersection(y[y == minority_class].index)
+        majority_idx = deduped_idx.difference(minority_idx)
+        n_majority_target = max(n_target - len(minority_idx), 0)
+        chosen_majority = rng.choice(majority_idx.to_numpy(),
+                                      size=min(n_majority_target, len(majority_idx)),
+                                      replace=False)
+        return minority_idx.append(pd.Index(chosen_majority))
+
+    def why_selected(self) -> str:
+        return "Dataset has a high duplicate-row percentage; deduplication is a safe first reduction."
+
+    def limitations(self) -> list[str]:
+        return ["Only effective when duplicate percentage is meaningfully high."]
+
+    @staticmethod
+    def fit_score(analysis: dict) -> float:
+        scan = analysis.get("scan", analysis)
+        dup_pct = scan.get("duplicate_pct", 0)
+        n_rows = scan.get("rows", 0)
+        if n_rows < 1000:
+            return 0.0
+        if dup_pct >= 5:
+            return 0.9
+        return 0.05

datapruning-1.0.7/datapruning/algorithms/instance_hardness.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import StratifiedKFold
+from sklearn.base import clone
+from sklearn.linear_model import LogisticRegression
+from datapruning.algorithms.base import BaseOptimizer
+from datapruning.algorithms.registry import register
+
+
+@register
+class InstanceHardnessOptimizer(BaseOptimizer):
+    key = "iht"
+    display_name = "Instance Hardness Threshold"
+
+    def __init__(self, estimator=None, n_folds: int = 5, random_state: int = 42):
+        self.estimator = estimator if estimator is not None else LogisticRegression(max_iter=1000)
+        self.n_folds = n_folds
+        self.random_state = random_state
+        self._hardness = None
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "InstanceHardnessOptimizer":
+        skf = StratifiedKFold(n_splits=self.n_folds, shuffle=True, random_state=self.random_state)
+        X_arr, y_arr = X.values, y.values
+        hardness = np.zeros(len(y_arr))
+        for train_idx, val_idx in skf.split(X_arr, y_arr):
+            model = clone(self.estimator)
+            model.fit(X_arr[train_idx], y_arr[train_idx])
+            probs = model.predict_proba(X_arr[val_idx])
+            class_to_col = {c: i for i, c in enumerate(model.classes_)}
+            true_probs = np.array(
+                [probs[i, class_to_col[y_arr[val_idx][i]]] for i in range(len(val_idx))]
+            )
+            hardness[val_idx] = 1.0 - true_probs
+        self._hardness = pd.Series(hardness, index=y.index)
+        return self
+
+    def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
+        if self._hardness is None:
+            self.fit(X, y)
+        minority_class = y.value_counts().idxmin()
+        minority_idx = y[y == minority_class].index
+        majority_idx = y[y != minority_class].index
+        n_total_target = int(len(y) * keep_ratio)
+        n_majority_target = max(n_total_target - len(minority_idx), 0)
+        majority_hardness = self._hardness.loc[majority_idx].sort_values(ascending=False)
+        kept_majority_idx = majority_hardness.index[:n_majority_target]
+        return minority_idx.append(kept_majority_idx)
+
+    def why_selected(self) -> str:
+        return ("Severe class imbalance detected. Instance Hardness Threshold removes "
+                "only the majority-class rows the model already classifies confidently, "
+                "preserving 100% of the minority class and all decision-boundary rows.")
+
+    def limitations(self) -> list[str]:
+        return ["Requires training a quick cross-validated model — slower than pure random.",
+                "Quality depends on the chosen base estimator; weak estimators give a noisier hardness signal."]
+
+    @staticmethod
+    def fit_score(analysis: dict) -> float:
+        scan = analysis.get("scan", analysis)
+        task_type = scan.get("task_type", "unknown")
+        imbalance_ratio = scan.get("imbalance_ratio", 1.0)
+        n_rows = scan.get("rows", 0)
+
+        if n_rows < 3000 or task_type == "regression":
+            return 0.0
+        if n_rows > 200_000:
+            return 0.3
+        if imbalance_ratio >= 10:
+            return 0.95
+        if imbalance_ratio >= 3:
+            return 0.7
+        return 0.4

datapruning-1.0.7/datapruning/algorithms/random_undersampler.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+import numpy as np
+import pandas as pd
+from datapruning.algorithms.base import BaseOptimizer
+from datapruning.algorithms.registry import register
+
+
+@register
+class RandomUnderSamplerOptimizer(BaseOptimizer):
+    key = "random"
+    display_name = "Random UnderSampler"
+
+    def __init__(self, random_state: int = 42):
+        self.random_state = random_state
+
+    def fit(self, X: pd.DataFrame, y: pd.Series) -> "RandomUnderSamplerOptimizer":
+        return self
+
+    def _select_rows(self, X: pd.DataFrame, y: pd.Series, keep_ratio: float) -> pd.Index:
+        rng = np.random.RandomState(self.random_state)
+        classes = y.unique()
+        minority_class = y.value_counts().idxmin()
+        minority_idx = y[y == minority_class].index
+        n_total_target = int(len(y) * keep_ratio)
+        n_majority_target = max(n_total_target - len(minority_idx), 0)
+        keep_idx = list(minority_idx)
+        for c in classes:
+            if c == minority_class:
+                continue
+            class_idx = y[y == c].index.to_numpy()
+            n_c_target = min(n_majority_target, len(class_idx))
+            chosen = rng.choice(class_idx, size=n_c_target, replace=False)
+            keep_idx.extend(chosen.tolist())
+        return pd.Index(keep_idx)
+
+    def why_selected(self) -> str:
+        return ("Large, balanced or mildly imbalanced dataset where blind random "
+                "reduction is acceptable and the fastest option.")
+
+    def limitations(self) -> list[str]:
+        return ["Removes rows with no regard for how informative they are.",
+                "Can discard rare-but-important patterns in the majority class."]
+
+    @staticmethod
+    def fit_score(analysis: dict) -> float:
+        scan = analysis.get("scan", analysis)
+        task_type = scan.get("task_type", "unknown")
+        imbalance_ratio = scan.get("imbalance_ratio", 1.0)
+        n_rows = scan.get("rows", 0)
+
+        if n_rows < 1000:
+            return 0.0
+        if task_type == "regression":
+            return 0.0
+        if imbalance_ratio < 3:
+            return 0.55
+        return 0.25

datapruning-1.0.7/datapruning/algorithms/registry.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+from typing import Type, Dict
+from datapruning.algorithms.base import BaseOptimizer
+
+
+class OptimizerRegistry:
+    def __init__(self):
+        self._registry: Dict[str, Type[BaseOptimizer]] = {}
+
+    def register(self, optimizer_cls: Type[BaseOptimizer]) -> Type[BaseOptimizer]:
+        if not optimizer_cls.key:
+            raise ValueError(f"{optimizer_cls.__name__} must set a non-empty `key`.")
+        self._registry[optimizer_cls.key] = optimizer_cls
+        return optimizer_cls
+
+    def get(self, key: str) -> BaseOptimizer:
+        if key not in self._registry:
+            raise KeyError(f"No optimizer registered under key '{key}'. "
+                           f"Available: {list(self._registry.keys())}")
+        return self._registry[key]()
+
+    def all(self) -> Dict[str, Type[BaseOptimizer]]:
+        return dict(self._registry)
+
+    def get_best(self, analysis: dict) -> tuple[BaseOptimizer, str, float, list[dict]]:
+        if not self._registry:
+            raise RuntimeError("No optimizers registered.")
+        scanned = []
+        for key, cls in self._registry.items():
+            score = cls.fit_score(analysis)
+            scanned.append((key, cls, score))
+        scanned.sort(key=lambda t: -t[2])
+        best_key, best_cls, best_score = scanned[0]
+        alternatives = [
+            {"key": k, "display_name": cls.display_name, "score": round(s, 3)}
+            for k, cls, s in scanned[1:]
+        ]
+        return best_cls(), best_key, best_score, alternatives
+
+
+registry = OptimizerRegistry()
+
+
+def register(cls: Type[BaseOptimizer]) -> Type[BaseOptimizer]:
+    return registry.register(cls)

datapruning-1.0.7/datapruning/engine/benchmark.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+import time
+import pandas as pd
+import numpy as np
+from datapruning.algorithms.base import BaseOptimizer
+from datapruning.algorithms.registry import registry
+
+
+def benchmark_algorithms(df: pd.DataFrame, target_col: str,
+                          algorithms: list[str] = None,
+                          keep_ratio: float = 0.5) -> dict:
+    if algorithms is None:
+        algorithms = list(registry.all().keys())
+    results = {}
+    for key in algorithms:
+        try:
+            optimizer = registry.get(key)
+            t0 = time.time()
+            result = optimizer.optimize(df, target_col=target_col, keep_ratio=keep_ratio)
+            elapsed = time.time() - t0
+            results[key] = {
+                "display_name": optimizer.display_name,
+                "rows_before": result.rows_before,
+                "rows_after": result.rows_after,
+                "reduction_pct": round((1 - result.rows_after / result.rows_before) * 100, 2) if result.rows_before else 0,
+                "runtime_seconds": round(elapsed, 3),
+                "minority_count_before": result.minority_count_before,
+                "minority_count_after": result.minority_count_after,
+            }
+        except Exception as e:
+            results[key] = {"display_name": key, "error": str(e)}
+    return results
+
+
+def compare_to_full(optimized_df: pd.DataFrame, original_df: pd.DataFrame,
+                     target_col: str) -> dict:
+    from datapruning.engine.scanner import scan
+    full_scan = scan(original_df, target_col=target_col)
+    opt_scan = scan(optimized_df, target_col=target_col)
+    return {
+        "original": {
+            "rows": full_scan["rows"],
+            "memory_mb": full_scan["memory_mb"],
+            "missing_pct": full_scan["missing_pct"],
+            "duplicate_pct": full_scan["duplicate_pct"],
+            "estimated_training_cost": full_scan["estimated_training_cost"],
+        },
+        "optimized": {
+            "rows": opt_scan["rows"],
+            "memory_mb": opt_scan["memory_mb"],
+            "missing_pct": opt_scan["missing_pct"],
+            "duplicate_pct": opt_scan["duplicate_pct"],
+            "estimated_training_cost": opt_scan["estimated_training_cost"],
+        },
+        "savings": {
+            "row_reduction_pct": round((1 - opt_scan["rows"] / full_scan["rows"]) * 100, 2) if full_scan["rows"] else 0,
+            "memory_saved_mb": round(full_scan["memory_mb"] - opt_scan["memory_mb"], 2),
+            "cost_reduction": _cost_reduction(full_scan["estimated_training_cost"], opt_scan["estimated_training_cost"]),
+        },
+    }
+
+
+def _cost_reduction(before: str, after: str) -> str:
+    rank = {"low": 0, "medium": 1, "high": 2, "very_high": 3}
+    diff = rank.get(before, 0) - rank.get(after, 0)
+    if diff >= 2:
+        return "significant"
+    elif diff == 1:
+        return "moderate"
+    return "minimal"

datapruning-1.0.7/datapruning/engine/explainability.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+import pandas as pd
+from datapruning.algorithms.base import BaseOptimizer, OptimizationResult, Explanation
+
+
+def build_report(optimizer: BaseOptimizer, result: OptimizationResult,
+                  df_before: pd.DataFrame) -> dict:
+    explanation: Explanation = optimizer.explain(result)
+    missing_pct_after = (result.optimized_df.isna().sum().sum() / result.optimized_df.size) * 100 \
+        if result.optimized_df.size else 0
+    dup_pct_after = (result.optimized_df.duplicated().sum() / len(result.optimized_df)) * 100 \
+        if len(result.optimized_df) else 0
+    quality = float(max(0, 100 - missing_pct_after * 2 - dup_pct_after * 2))
+    size_reduction = float(1 - (result.rows_after / result.rows_before)) if result.rows_before else 0.0
+    training_readiness = float(min(100, 70 + size_reduction * 60))
+    overall_health = round((quality + training_readiness) / 2, 1)
+    return {
+        "explanation": explanation.__dict__,
+        "dataset_score": {
+            "overall_health": overall_health,
+            "quality": round(quality, 1),
+            "training_readiness": round(training_readiness, 1),
+            "optimization_potential": round(size_reduction * 100, 1),
+        },
+        "runtime_seconds": round(result.runtime_seconds, 3),
+    }

datapruning-1.0.7/datapruning/engine/intelligence.py

@@ -0,0 +1,119 @@
+import numpy as np
+import pandas as pd
+
+
+def analyze_intelligence(df: pd.DataFrame, target_col: str = None) -> dict:
+    findings = {
+        "id_columns": _detect_id_columns(df),
+        "date_columns": _detect_date_columns(df),
+        "constant_columns": _detect_constant_columns(df),
+        "near_constant_columns": _detect_near_constant_columns(df),
+        "high_cardinality_columns": _detect_high_cardinality(df),
+        "highly_correlated_pairs": _detect_correlated_features(df, target_col),
+        "outlier_summary": _detect_outliers(df),
+        "complexity_score": None,
+        "leakage_risk_columns": [],
+    }
+    if target_col and target_col in df.columns:
+        findings["leakage_risk_columns"] = _detect_leakage_risk(df, target_col)
+    findings["complexity_score"] = _complexity_score(df, findings)
+    return findings
+
+
+def _detect_id_columns(df, uniqueness_threshold=0.98):
+    out = []
+    for col in df.columns:
+        dtype = df[col].dtype
+        is_id_like_dtype = pd.api.types.is_integer_dtype(dtype) or dtype == object
+        if is_id_like_dtype and df[col].nunique() / len(df) >= uniqueness_threshold:
+            out.append(col)
+    return out
+
+
+def _detect_date_columns(df):
+    out = []
+    for col in df.select_dtypes(include=["object", "datetime64[ns]"]).columns:
+        if pd.api.types.is_datetime64_any_dtype(df[col]):
+            out.append(col)
+            continue
+        sample = df[col].dropna().head(50)
+        try:
+            parsed = pd.to_datetime(sample, errors="coerce")
+            if parsed.notna().mean() > 0.9:
+                out.append(col)
+        except Exception:
+            pass
+    return out
+
+
+def _detect_constant_columns(df):
+    return [c for c in df.columns if df[c].nunique(dropna=False) <= 1]
+
+
+def _detect_near_constant_columns(df, threshold=0.98):
+    out = []
+    for c in df.columns:
+        top_freq = df[c].value_counts(normalize=True, dropna=False).iloc[0]
+        if top_freq >= threshold and df[c].nunique() > 1:
+            out.append(c)
+    return out
+
+
+def _detect_high_cardinality(df, threshold=0.5):
+    out = []
+    for c in df.select_dtypes(include=["object", "category"]).columns:
+        if df[c].nunique() / len(df) >= threshold:
+            out.append(c)
+    return out
+
+
+def _detect_correlated_features(df, target_col=None, threshold=0.95):
+    numeric_df = df.select_dtypes(include=[np.number])
+    if target_col in numeric_df.columns:
+        numeric_df = numeric_df.drop(columns=[target_col])
+    if numeric_df.shape[1] < 2:
+        return []
+    corr = numeric_df.corr().abs()
+    pairs = []
+    cols = corr.columns
+    for i in range(len(cols)):
+        for j in range(i + 1, len(cols)):
+            val = corr.iloc[i, j]
+            if val >= threshold:
+                pairs.append({"col_a": cols[i], "col_b": cols[j], "correlation": round(float(val), 3)})
+    return sorted(pairs, key=lambda x: -x["correlation"])[:25]
+
+
+def _detect_outliers(df, z_thresh=4.0):
+    numeric_df = df.select_dtypes(include=[np.number])
+    summary = {}
+    for c in numeric_df.columns:
+        col = numeric_df[c].dropna()
+        if col.std() == 0 or len(col) == 0:
+            continue
+        z = (col - col.mean()) / col.std()
+        pct = (z.abs() > z_thresh).mean() * 100
+        if pct > 0:
+            summary[c] = round(float(pct), 2)
+    return summary
+
+
+def _detect_leakage_risk(df, target_col, corr_threshold=0.98):
+    if target_col not in df.columns:
+        return []
+    numeric_df = df.select_dtypes(include=[np.number])
+    if target_col not in numeric_df.columns or numeric_df.shape[1] < 2:
+        return []
+    corrs = numeric_df.corr()[target_col].drop(target_col).abs()
+    return [c for c, v in corrs.items() if v >= corr_threshold]
+
+
+def _complexity_score(df, findings) -> int:
+    score = 0
+    n_cols = df.shape[1]
+    score += min(len(findings["constant_columns"]) / max(n_cols, 1) * 100, 20)
+    score += min(len(findings["near_constant_columns"]) / max(n_cols, 1) * 100, 15)
+    score += min(len(findings["highly_correlated_pairs"]) * 2, 20)
+    score += min(len(findings["high_cardinality_columns"]) / max(n_cols, 1) * 100, 15)
+    score += min(len(findings["leakage_risk_columns"]) * 15, 30)
+    return int(round(min(score, 100)))

datapruning-1.0.7/datapruning/engine/optimizer.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+import pandas as pd
+from datapruning.algorithms.base import BaseOptimizer, OptimizationResult
+
+
+def run_optimization(df: pd.DataFrame, target_col: str, optimizer: BaseOptimizer,
+                      keep_ratio: float = 0.5) -> OptimizationResult:
+    return optimizer.optimize(df, target_col=target_col, keep_ratio=keep_ratio)

datapruning-1.0.7/datapruning/engine/scanner.py

@@ -0,0 +1,48 @@
+import numpy as np
+import pandas as pd
+
+
+def scan(df: pd.DataFrame, target_col: str = None) -> dict:
+    n_rows, n_cols = df.shape
+    memory_mb = df.memory_usage(deep=True).sum() / 1e6
+    missing_pct = (df.isna().sum().sum() / (n_rows * n_cols)) * 100
+    dup_pct = (df.duplicated().sum() / n_rows) * 100
+    numeric_cols = df.select_dtypes(include=[np.number]).columns
+    categorical_cols = df.select_dtypes(exclude=[np.number]).columns
+    result = {
+        "rows": n_rows,
+        "columns": n_cols,
+        "memory_mb": round(memory_mb, 2),
+        "missing_pct": round(missing_pct, 3),
+        "duplicate_pct": round(dup_pct, 3),
+        "numeric_pct": round(len(numeric_cols) / n_cols * 100, 1),
+        "categorical_pct": round(len(categorical_cols) / n_cols * 100, 1),
+        "estimated_training_cost": _estimate_cost(n_rows, n_cols),
+    }
+    if target_col and target_col in df.columns:
+        result["task_type"] = _infer_task_type(df[target_col])
+        if result["task_type"] in ("binary", "multiclass"):
+            vc = df[target_col].value_counts(normalize=True)
+            result["target_balance"] = {str(k): round(v * 100, 2) for k, v in vc.items()}
+            result["imbalance_ratio"] = round(vc.max() / vc.min(), 2) if len(vc) > 1 else 1.0
+    return result
+
+
+def _infer_task_type(y: pd.Series) -> str:
+    nunique = y.nunique()
+    if pd.api.types.is_numeric_dtype(y) and nunique > 20:
+        return "regression"
+    if nunique == 2:
+        return "binary"
+    return "multiclass"
+
+
+def _estimate_cost(n_rows: int, n_cols: int) -> str:
+    scale = n_rows * n_cols
+    if scale < 1e6:
+        return "low"
+    elif scale < 5e7:
+        return "medium"
+    elif scale < 5e8:
+        return "high"
+    return "very_high"

datapruning-1.0.7/datapruning/engine/strategy_selector.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+from dataclasses import dataclass
+import datapruning.algorithms
+from datapruning.algorithms.registry import registry
+
+
+@dataclass
+class StrategyRecommendation:
+    algorithm_key: str
+    algorithm_instance: object
+    display_name: str
+    confidence: float
+    alternatives: list[dict]
+
+
+def select_strategy(analysis: dict) -> StrategyRecommendation:
+    best_instance, best_key, best_score, alternatives = registry.get_best(analysis)
+    return StrategyRecommendation(
+        algorithm_key=best_key,
+        algorithm_instance=best_instance,
+        display_name=best_instance.display_name,
+        confidence=round(best_score, 3),
+        alternatives=alternatives,
+    )
+
+
+def get_algorithm(key: str):
+    return registry.get(key)
+
+
+def list_available_algorithms() -> list[dict]:
+    return [
+        {"key": key, "display_name": cls.display_name}
+        for key, cls in registry.all().items()
+    ]

{datapruning-1.0.6 → datapruning-1.0.7}/datapruning/pipeline.py

@@ -1,7 +1,6 @@
 import logging, re
 import numpy as np
 import pandas as pd
-import datapruning._core as _core
 
 logger = logging.getLogger(__name__)
 

datapruning-1.0.7/datapruning/reports/exporter.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+from pathlib import Path
+import pandas as pd
+
+
+def export_csv(df: pd.DataFrame, path: str) -> str:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    df.to_csv(path, index=False)
+    return str(path)
+
+
+def export_parquet(df: pd.DataFrame, path: str) -> str:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    df.to_parquet(path, index=False)
+    return str(path)
+
+
+def export_json(df: pd.DataFrame, path: str, orient: str = "records") -> str:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    df.to_json(path, orient=orient, indent=2)
+    return str(path)

datapruning-1.0.7/datapruning/sdk.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+import json
+import pandas as pd
+from datapruning.engine.scanner import scan
+from datapruning.engine.intelligence import analyze_intelligence
+from datapruning.engine.strategy_selector import select_strategy, get_algorithm, list_available_algorithms
+from datapruning.engine.optimizer import run_optimization
+from datapruning.engine.explainability import build_report
+from datapruning.engine.benchmark import benchmark_algorithms, compare_to_full
+
+
+class DatasetEngine:
+    def __init__(self, df: pd.DataFrame, target_col: str = None):
+        self.df = df
+        self.target_col = target_col
+        self._analysis = None
+        self._last_result = None
+        self._last_optimizer = None
+
+    def analyze(self) -> dict:
+        scan_result = scan(self.df, target_col=self.target_col)
+        intelligence_result = analyze_intelligence(self.df, target_col=self.target_col)
+        self._analysis = {"scan": scan_result, "intelligence": intelligence_result}
+        return self._analysis
+
+    def report(self, as_json: bool = False):
+        if self._analysis is None:
+            self.analyze()
+        if as_json:
+            return json.dumps(self._analysis, indent=2, default=str)
+        s, i = self._analysis["scan"], self._analysis["intelligence"]
+        print(f"Rows: {s['rows']:,} | Columns: {s['columns']} | Memory: {s['memory_mb']} MB")
+        print(f"Missing: {s['missing_pct']}% | Duplicates: {s['duplicate_pct']}%")
+        if "task_type" in s:
+            print(f"Task type: {s['task_type']}")
+            if "imbalance_ratio" in s:
+                print(f"Imbalance ratio: {s['imbalance_ratio']}:1")
+        if i["leakage_risk_columns"]:
+            print(f"Possible leakage columns: {i['leakage_risk_columns']}")
+        print(f"Complexity score: {i['complexity_score']}/100")
+        return self._analysis
+
+    def recommend(self):
+        if self._analysis is None:
+            self.analyze()
+        return select_strategy(self._analysis)
+
+    def available_algorithms(self) -> list[dict]:
+        return list_available_algorithms()
+
+    def optimize(self, algorithm_key: str = None, keep_ratio: float = 0.5) -> dict:
+        if self.target_col is None:
+            raise ValueError("target_col is required to optimize.")
+        if algorithm_key is None:
+            rec = self.recommend()
+            optimizer = rec.algorithm_instance
+        else:
+            optimizer = get_algorithm(algorithm_key)
+        result = run_optimization(self.df, self.target_col, optimizer, keep_ratio=keep_ratio)
+        report = build_report(optimizer, result, self.df)
+        self._last_result = result
+        self._last_optimizer = optimizer
+        return {
+            "optimized_df": result.optimized_df,
+            "explainability": report["explanation"],
+            "dataset_score": report["dataset_score"],
+            "runtime_seconds": report["runtime_seconds"],
+        }
+
+    def export(self, path: str, result: dict = None):
+        result = result or {"optimized_df": self._last_result.optimized_df}
+        result["optimized_df"].to_csv(path, index=False)
+        print(f"Exported optimized dataset -> {path} ({len(result['optimized_df']):,} rows)")
+        return path
+
+    def benchmark(self, algorithms: list[str] = None, keep_ratio: float = 0.5) -> dict:
+        return benchmark_algorithms(self.df, self.target_col, algorithms=algorithms, keep_ratio=keep_ratio)
+
+    def compare(self, optimized_df: pd.DataFrame) -> dict:
+        return compare_to_full(optimized_df, self.df, self.target_col)

{datapruning-1.0.6 → datapruning-1.0.7/datapruning.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 1.0.6
+Version: 1.0.7
 Summary: Intelligent data pruning for ML datasets
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com
@@ -16,9 +16,12 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: numpy>=1.24.0
 Requires-Dist: pandas>=2.0.0
+Requires-Dist: scikit-learn>=1.3.0
 Requires-Dist: torch>=2.0.0
+Dynamic: license-file
 
 # DataPruning
 

datapruning-1.0.7/datapruning.egg-info/SOURCES.txt

@@ -0,0 +1,31 @@
+LICENSE
+MANIFEST.in
+README.md
+pyproject.toml
+setup.py
+build_src/_core.c
+datapruning/__init__.py
+datapruning/pipeline.py
+datapruning/sdk.py
+datapruning.egg-info/PKG-INFO
+datapruning.egg-info/SOURCES.txt
+datapruning.egg-info/dependency_links.txt
+datapruning.egg-info/requires.txt
+datapruning.egg-info/top_level.txt
+datapruning/algorithms/__init__.py
+datapruning/algorithms/base.py
+datapruning/algorithms/cluster_centroids.py
+datapruning/algorithms/density_pruner.py
+datapruning/algorithms/duplicate_aware.py
+datapruning/algorithms/instance_hardness.py
+datapruning/algorithms/random_undersampler.py
+datapruning/algorithms/registry.py
+datapruning/engine/__init__.py
+datapruning/engine/benchmark.py
+datapruning/engine/explainability.py
+datapruning/engine/intelligence.py
+datapruning/engine/optimizer.py
+datapruning/engine/scanner.py
+datapruning/engine/strategy_selector.py
+datapruning/reports/__init__.py
+datapruning/reports/exporter.py

{datapruning-1.0.6 → datapruning-1.0.7}/datapruning.egg-info/requires.txt

@@ -1,3 +1,4 @@
 numpy>=1.24.0
 pandas>=2.0.0
+scikit-learn>=1.3.0
 torch>=2.0.0

{datapruning-1.0.6 → datapruning-1.0.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datapruning"
-version = "1.0.6"
+version = "1.0.7"
 description = "Intelligent data pruning for ML datasets"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -24,14 +24,12 @@ classifiers = [
 dependencies = [
     "numpy>=1.24.0",
     "pandas>=2.0.0",
+    "scikit-learn>=1.3.0",
     "torch>=2.0.0",
 ]
 
 [project.urls]
 Homepage = "https://www.datapruning.com"
 
-[tool.setuptools]
-license-files = []
-
 [tool.setuptools.packages.find]
 include = ["datapruning", "datapruning.*"]

datapruning-1.0.6/datapruning.egg-info/SOURCES.txt

@@ -1,13 +0,0 @@
-LICENSE
-MANIFEST.in
-README.md
-pyproject.toml
-setup.py
-build_src/_core.c
-datapruning/__init__.py
-datapruning/pipeline.py
-datapruning.egg-info/PKG-INFO
-datapruning.egg-info/SOURCES.txt
-datapruning.egg-info/dependency_links.txt
-datapruning.egg-info/requires.txt
-datapruning.egg-info/top_level.txt