datapruning 1.0.3__cp314-cp314-win_amd64.whl

datapruning/__init__.py

@@ -0,0 +1,61 @@
+import logging
+import numpy as np
+import pandas as pd
+
+from . import pipeline
+from ._core import analyze_data as _analyze_data
+
+logger = logging.getLogger(__name__)
+
+MIN_ROWS = 1000
+MAX_ROWS = 300_000
+
+
+def prune_dataframe(df, target_col=None, keep_ratio=0.5, num_steps=400, verbose=False):
+    if verbose:
+        logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+
+    if len(df) < MIN_ROWS:
+        raise ValueError(
+            f"Dataset too small for pruning. Minimum {MIN_ROWS:,} rows required, "
+            f"got {len(df):,}."
+        )
+    if len(df) > MAX_ROWS:
+        raise ValueError(
+            f"Dataset exceeds maximum of {MAX_ROWS:,} rows (got {len(df):,})."
+        )
+
+    if target_col is None:
+        target_col = pipeline.detect_target_column(df)
+        logger.info("Auto-detected target column: '%s'", target_col)
+
+    if target_col not in df.columns:
+        raise ValueError(f"Column '{target_col}' not found in DataFrame")
+
+    info = pipeline.analyze_dataset(df, target_col)
+    logger.info("Task: %s, %d rows, %d features, %d classes",
+                info["task_type"], info["row_count"],
+                info["feature_count"], info["num_classes"] or "N/A")
+
+    X, y, meta = pipeline.translate_df(df, target_col)
+
+    X_f32 = np.ascontiguousarray(X, dtype=np.float32)
+    indices = _analyze_data(X_f32, y.copy(), num_steps, keep_ratio)
+
+    indices, adjustments = pipeline.safety_layer(indices, y, original_count=len(df))
+    if adjustments:
+        logger.info("Safety adjustments: %s", adjustments)
+
+    return df.iloc[indices].reset_index(drop=True)
+
+
+class Pruner:
+    def __init__(self, df):
+        if not isinstance(df, pd.DataFrame):
+            raise TypeError("Expected a pandas DataFrame")
+        self._df = df
+
+    def prune(self, target_col=None, keep_ratio=0.5, num_steps=400, verbose=False):
+        return prune_dataframe(self._df, target_col=target_col,
+                               keep_ratio=keep_ratio, num_steps=num_steps,
+                               verbose=verbose)

datapruning/pipeline.py

@@ -0,0 +1,188 @@
+import logging, re
+import numpy as np
+import pandas as pd
+import datapruning._core as _core
+
+logger = logging.getLogger(__name__)
+
+SMALL_DATASET_THRESHOLD = 3000
+HIGH_DIM_THRESHOLD = 50
+IMBALANCE_RATIO_THRESHOLD = 0.1
+MAX_PRUNING = 0.60
+MIN_ROWS = 1000
+MAX_ROWS = 300_000
+
+SUSPICIOUS_NAMES = [
+    "target", "class", "label", "outcome",
+    "y", "diagnosis", "income",
+    "category", "prediction",
+]
+BOOL_MAPS = {
+    frozenset({"yes", "no"}): {"yes": 1, "no": 0},
+    frozenset({"true", "false"}): {"true": 1, "false": 0},
+    frozenset({"male", "female"}): {"male": 1, "female": 0},
+    frozenset({"y", "n"}): {"y": 1, "n": 0},
+}
+
+
+def detect_target_column(df):
+    candidates = []
+    for col in df.columns:
+        score = 0.0
+        series = df[col]
+        unique_count = series.nunique(dropna=True)
+        unique_ratio = unique_count / max(len(series), 1)
+        is_numeric = pd.api.types.is_numeric_dtype(series)
+        col_lower = str(col).lower()
+        tokens = set(re.split(r'[_\-\.\s]+', col_lower))
+        if tokens & set(SUSPICIOUS_NAMES):
+            score += 50
+        if unique_count <= 20:
+            score += 35
+        elif unique_count <= 50:
+            score += 15
+        if unique_count == 2:
+            score += 25
+        if unique_ratio > 0.9:
+            score -= 40
+        if "id" in col_lower:
+            score -= 50
+        if "date" in col_lower or "time" in col_lower:
+            score -= 30
+        if is_numeric and unique_count > 100:
+            score -= 20
+        if col == df.columns[0]:
+            score += 10
+        if col == df.columns[-1]:
+            score += 15
+        candidates.append({"column": col, "score": score})
+    candidates.sort(key=lambda x: x["score"], reverse=True)
+    return candidates[0]["column"]
+
+
+def translate_df(df, target_column):
+    feature_names = [c for c in df.columns if c != target_column]
+    target_name = target_column
+
+    df = df.dropna(subset=[target_name]).copy()
+    if df.empty:
+        raise ValueError("All rows have missing target values")
+
+    features = df[feature_names].copy()
+
+    for col in features.columns:
+        uniq = set(features[col].dropna().astype(str).str.lower().unique())
+        for pattern, mapping in BOOL_MAPS.items():
+            if uniq and (uniq == pattern or uniq.issubset(pattern)):
+                features[col] = features[col].astype(str).str.lower().map(mapping).fillna(0).astype(int)
+                break
+
+    num_cols = features.select_dtypes(include=[np.number]).columns.tolist()
+    cat_cols = [c for c in features.columns if c not in num_cols]
+
+    for col in num_cols:
+        features[col] = pd.to_numeric(features[col], errors="coerce")
+        med = features[col].median()
+        features[col] = features[col].fillna(med if pd.notna(med) else 0.0)
+
+    for col in cat_cols:
+        features[col] = features[col].astype(str).replace("nan", pd.NA)
+        most_common = features[col].mode()
+        fill_val = most_common.iloc[0] if not most_common.empty else "missing"
+        features[col] = features[col].fillna(fill_val)
+
+    if cat_cols:
+        features = pd.get_dummies(features, columns=cat_cols, prefix=cat_cols, dummy_na=False)
+
+    for col in features.columns:
+        features[col] = pd.to_numeric(features[col], errors="coerce").fillna(0.0)
+
+    target_series = df[target_name].fillna("missing").astype(str)
+    classes = sorted(target_series.unique())
+    le = {v: i for i, v in enumerate(classes)}
+    y = target_series.map(le).fillna(-1).astype(np.int64).values
+    X = features.values.astype(np.float64)
+
+    metadata = {
+        "feature_names": features.columns.tolist(),
+        "target_name": target_name,
+        "num_features": X.shape[1],
+        "num_classes": len(classes),
+        "class_mapping": le,
+    }
+
+    return X, y, metadata
+
+
+def analyze_dataset(df, target_column):
+    feature_count = df.shape[1] - 1
+    row_count = len(df)
+    target_series = df[target_column].dropna()
+    unique_classes = target_series.unique()
+    num_classes = len(unique_classes)
+    is_numeric_target = pd.api.types.is_numeric_dtype(target_series)
+    unique_ratio = num_classes / max(len(target_series), 1)
+
+    if num_classes <= 20 and (not is_numeric_target or unique_ratio < 0.01):
+        task_type = "classification"
+    elif is_numeric_target and num_classes > 20:
+        task_type = "regression"
+    else:
+        task_type = "classification"
+
+    risks = []
+    if row_count < SMALL_DATASET_THRESHOLD:
+        risks.append({"type": "small_dataset", "message": "Small dataset detected.", "severity": "info"})
+    if feature_count >= HIGH_DIM_THRESHOLD:
+        risks.append({"type": "high_dimensional", "message": "High-dimensional dataset detected.", "severity": "info"})
+
+    has_imbalance = False
+    class_distribution = None
+    if task_type == "classification" and num_classes >= 2:
+        counts = target_series.value_counts()
+        class_distribution = {str(k): int(v) for k, v in counts.items()}
+        min_count = counts.min()
+        max_count = counts.max()
+        if min_count > 0 and (min_count / max_count) < IMBALANCE_RATIO_THRESHOLD:
+            has_imbalance = True
+            risks.append({"type": "imbalance", "message": "Class imbalance detected.", "severity": "info"})
+
+    return {
+        "row_count": row_count,
+        "feature_count": feature_count,
+        "task_type": task_type,
+        "num_classes": num_classes if task_type == "classification" else None,
+        "class_distribution": class_distribution,
+        "has_imbalance": has_imbalance,
+        "risks": risks,
+    }
+
+
+def safety_layer(kept_indices, y, original_count=None, min_per_class=3):
+    adjustments = []
+
+    if original_count is not None:
+        min_kept = max(1, int(original_count * (1 - MAX_PRUNING)))
+        if len(kept_indices) < min_kept:
+            kept_set = set(kept_indices)
+            all_set = set(range(original_count))
+            removed = all_set - kept_set
+            to_add = sorted(removed)[:min_kept - len(kept_indices)]
+            kept_indices = np.array(sorted(kept_set | set(to_add)))
+            adjustments.append("max_pruning_cap_applied")
+
+    kept_set = set(kept_indices)
+    unique_classes = np.unique(y)
+    result = list(kept_indices)
+    for cls in unique_classes:
+        cls_indices = np.where(y == cls)[0]
+        cls_kept = [int(i) for i in cls_indices if i in kept_set]
+        if len(cls_kept) < min_per_class:
+            to_add = min_per_class - len(cls_kept)
+            candidates = [int(i) for i in cls_indices if i not in kept_set]
+            result.extend(candidates[:to_add])
+    if len(result) != len(kept_indices):
+        adjustments.append("class_preservation_applied")
+    kept_indices = np.array(sorted(result))
+
+    return kept_indices, adjustments

datapruning-1.0.3.dist-info/METADATA

@@ -0,0 +1,73 @@
+Metadata-Version: 2.4
+Name: datapruning
+Version: 1.0.3
+Summary: Intelligent data pruning for ML datasets
+License: Proprietary
+Project-URL: Homepage, https://www.datapruning.com
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: torch>=2.0.0
+
+# DataPruning
+
+Intelligent dataset pruning for ML — reduces dataset size by selecting the most informative rows.
+
+## Installation
+
+```bash
+pip install datapruning
+```
+
+**Note:** PyTorch (~2 GB) is a required dependency.
+
+## Requirements
+
+- Python >= 3.10
+- PyTorch 2.0+
+- Pandas 2.0+
+- NumPy 1.24+
+
+## Limits
+
+- Minimum: 1,000 rows per dataset
+- Maximum: 300,000 rows per dataset
+
+## Usage
+
+```python
+import pandas as pd
+from datapruning import Pruner
+
+df = pd.read_csv("dataset.csv")
+p = Pruner(df)
+filtered = p.prune(target_col="label", keep_ratio=0.5)
+
+print(f"Kept {len(filtered)} / {len(df)} rows")
+```
+
+## Features
+
+- Runs locally, no data upload
+- Pre-training dataset optimization
+- Pandas DataFrame input / output
+- Compiled processing module
+
+## Links
+
+- Website: [datapruning.com](https://www.datapruning.com)
+
+## License
+
+Proprietary. See LICENSE file.

datapruning-1.0.3.dist-info/RECORD

@@ -0,0 +1,7 @@
+datapruning/__init__.py,sha256=4iQ1n8K0FTPSwbC1T3Oh4oFfzwkQI9H5ac3h_N3kwGY,2127
+datapruning/_core.cp314-win_amd64.pyd,sha256=TvIKgYUtOxS-Euwh60H36gauUUB4kDY1BeQraFNgHoI,348160
+datapruning/pipeline.py,sha256=T78LTTq5PFSfW7S4rKPXvupdEenkFK9tXW6AkHY8lDM,6973
+datapruning-1.0.3.dist-info/METADATA,sha256=dCenCUNPq2MNO6ds5r8kA53O0KysD0Bbo6NIkNaXKEE,1779
+datapruning-1.0.3.dist-info/WHEEL,sha256=jhxrRP3tdEfm7toB1X7dl5x6f0bu6Tve2cWwIdkbL8U,101
+datapruning-1.0.3.dist-info/top_level.txt,sha256=HHitOoBY_0KeZGXmSViQTKsbSMkUJNPSc7oUMAoxlF8,12
+datapruning-1.0.3.dist-info/RECORD,,

datapruning-1.0.3.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: false
+Tag: cp314-cp314-win_amd64
+

datapruning-1.0.3.dist-info/top_level.txt

@@ -0,0 +1 @@
+datapruning