datapruning 2.0.0__tar.gz → 2.1.0__tar.gz

{datapruning-2.0.0/datapruning.egg-info → datapruning-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 2.0.0
+Version: 2.1.0
 Summary: Intelligent data pruning for ML datasets
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com
@@ -21,6 +21,9 @@ Requires-Dist: numpy>=1.24.0
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: scikit-learn>=1.3.0
 Requires-Dist: torch>=2.0.0
+Provides-Extra: clip
+Requires-Dist: transformers>=4.30.0; extra == "clip"
+Requires-Dist: pillow>=9.0.0; extra == "clip"
 Dynamic: license-file
 
 # DataPruning

datapruning-2.1.0/datapruning/multimodal/__init__.py

@@ -0,0 +1 @@
+"""Multimodal pruning support — encode images and text with CLIP, then prune."""

datapruning-2.1.0/datapruning/multimodal/clip_encoder.py

@@ -0,0 +1,179 @@
+"""CLIP-based encoder for images and text.
+
+Turns images and text into feature vectors that the pruning algorithms
+can work with. Uses HuggingFace transformers for CLIP.
+
+Requirements:
+    pip install datapruning[clip]
+    # or manually: pip install transformers pillow
+"""
+
+from __future__ import annotations
+
+import os
+import logging
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+CLIP_MODEL = "openai/clip-vit-base-patch32"
+IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".bmp", ".gif", ".tiff", ".webp"}
+
+try:
+    import torch
+    from transformers import CLIPModel, CLIPTokenizer, CLIPProcessor
+
+    HAS_CLIP = True
+except ImportError:
+    HAS_CLIP = False
+
+
+def _require_clip():
+    if not HAS_CLIP:
+        raise ImportError(
+            "CLIP dependencies not installed. "
+            "Install with: pip install datapruning[clip]  "
+            "or manually:  pip install transformers pillow"
+        )
+
+
+def _to_tensor(x):
+    """Extract a torch.Tensor from various CLIP output types."""
+    if isinstance(x, torch.Tensor):
+        return x
+    if hasattr(x, "pooler_output") and x.pooler_output is not None:
+        return x.pooler_output
+    if hasattr(x, "last_hidden_state"):
+        return x.last_hidden_state[:, 0, :]
+    if hasattr(x, "text_embeds"):
+        return x.text_embeds
+    if hasattr(x, "image_embeds"):
+        return x.image_embeds
+    raise TypeError(f"Cannot extract tensor from {type(x).__name__}")
+
+
+class CLIPEncoder:
+    """Encode images and text into CLIP feature vectors."""
+
+    def __init__(self, model_name: str = CLIP_MODEL, device: Optional[str] = None):
+        _require_clip()
+        self.model_name = model_name
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+        self._model = None
+        self._processor = None
+        self._tokenizer = None
+
+    def _load(self):
+        if self._model is not None:
+            return
+        logger.info("Loading CLIP model: %s (device=%s)", self.model_name, self.device)
+        self._model = CLIPModel.from_pretrained(self.model_name).to(self.device)
+        self._model.eval()
+        self._processor = CLIPProcessor.from_pretrained(self.model_name)
+        self._tokenizer = CLIPTokenizer.from_pretrained(self.model_name)
+        logger.info("CLIP model loaded successfully")
+
+    def encode_images(
+        self, image_paths: list[str], batch_size: int = 32
+    ) -> np.ndarray:
+        """Encode a list of image file paths into CLIP embeddings.
+
+        Parameters
+        ----------
+        image_paths : list of str — paths to image files.
+        batch_size : int — encoding batch size.
+
+        Returns
+        -------
+        embeddings : (N, 512) float32 array — CLIP image features.
+        """
+        _require_clip()
+        self._load()
+
+        from PIL import Image
+
+        all_embeddings = []
+        for i in range(0, len(image_paths), batch_size):
+            batch_paths = image_paths[i : i + batch_size]
+            images = []
+            for p in batch_paths:
+                try:
+                    img = Image.open(p).convert("RGB")
+                    images.append(img)
+                except Exception as e:
+                    logger.warning("Failed to load %s: %s", p, e)
+                    images.append(Image.new("RGB", (224, 224), (0, 0, 0)))
+
+            inputs = self._processor(images=images, return_tensors="pt", padding=True)
+            inputs = {k: v.to(self.device) for k, v in inputs.items()}
+
+            with torch.no_grad():
+                features = self._model.get_image_features(**inputs)
+                features = _to_tensor(features)
+                features = features / features.norm(dim=-1, keepdim=True)
+
+            all_embeddings.append(features.cpu().numpy().astype(np.float32))
+
+        return np.concatenate(all_embeddings, axis=0)
+
+    def encode_texts(
+        self, texts: list[str], batch_size: int = 64
+    ) -> np.ndarray:
+        """Encode a list of text strings into CLIP embeddings.
+
+        Parameters
+        ----------
+        texts : list of str — text strings to encode.
+        batch_size : int — encoding batch size.
+
+        Returns
+        -------
+        embeddings : (N, 512) float32 array — CLIP text features.
+        """
+        _require_clip()
+        self._load()
+
+        all_embeddings = []
+        for i in range(0, len(texts), batch_size):
+            batch = texts[i : i + batch_size]
+            inputs = self._tokenizer(
+                batch, padding=True, truncation=True, return_tensors="pt"
+            )
+            inputs = {k: v.to(self.device) for k, v in inputs.items()}
+
+            with torch.no_grad():
+                features = self._model.get_text_features(**inputs)
+                features = _to_tensor(features)
+                features = features / features.norm(dim=-1, keepdim=True)
+
+            all_embeddings.append(features.cpu().numpy().astype(np.float32))
+
+        return np.concatenate(all_embeddings, axis=0)
+
+
+def scan_images(folder_path: str) -> list[str]:
+    """Recursively find all image files in a folder.
+
+    Returns sorted list of absolute paths.
+    """
+    folder = Path(folder_path)
+    if not folder.exists():
+        raise FileNotFoundError(f"Folder not found: {folder_path}")
+    if not folder.is_dir():
+        raise NotADirectoryError(f"Not a directory: {folder_path}")
+
+    paths = []
+    for ext in IMAGE_EXTENSIONS:
+        paths.extend(folder.rglob(f"*{ext}"))
+        paths.extend(folder.rglob(f"*{ext.upper()}"))
+
+    paths = sorted(set(str(p) for p in paths))
+    if not paths:
+        raise ValueError(
+            f"No image files found in {folder_path}. "
+            f"Supported formats: {', '.join(sorted(IMAGE_EXTENSIONS))}"
+        )
+    return paths

datapruning-2.1.0/datapruning/sdk.py

@@ -0,0 +1,289 @@
+from __future__ import annotations
+import json
+import logging
+import numpy as np
+import pandas as pd
+from pathlib import Path
+from datapruning.engine.scanner import scan
+from datapruning.engine.intelligence import analyze_intelligence
+from datapruning.engine.strategy_selector import select_strategy, get_algorithm, list_available_algorithms
+from datapruning.engine.optimizer import run_optimization
+from datapruning.engine.explainability import build_report
+from datapruning.engine.benchmark import benchmark_algorithms, compare_to_full
+
+logger = logging.getLogger(__name__)
+
+
+class DatasetEngine:
+    def __init__(self, df: pd.DataFrame, target_col: str = None):
+        self.df = df
+        self.target_col = target_col
+        self._analysis = None
+        self._last_result = None
+        self._last_optimizer = None
+        self._embeddings = None
+        self._input_mode = "tabular"
+
+    @classmethod
+    def from_images(
+        cls,
+        folder_path: str,
+        labels: list[int],
+        keep_ratio: float = 0.5,
+        batch_size: int = 32,
+    ) -> "DatasetEngine":
+        """Create a DatasetEngine from a folder of images.
+
+        Encodes all images with CLIP into feature vectors, then wraps
+        them in a DataFrame so the existing pruning algorithms work
+        unchanged.
+
+        Parameters
+        ----------
+        folder_path : str — path to folder containing images.
+        labels : list of int — integer label for each image (same order).
+        keep_ratio : float — default keep ratio for optimize().
+        batch_size : int — CLIP encoding batch size.
+
+        Returns
+        -------
+        DatasetEngine — ready to call analyze(), recommend(), optimize().
+        """
+        from datapruning.multimodal.clip_encoder import CLIPEncoder, scan_images
+
+        image_paths = scan_images(folder_path)
+        if len(image_paths) != len(labels):
+            raise ValueError(
+                f"Number of images ({len(image_paths)}) != "
+                f"number of labels ({len(labels)})"
+            )
+
+        encoder = CLIPEncoder()
+        embeddings = encoder.encode_images(image_paths, batch_size=batch_size)
+
+        feat_cols = [f"clip_{i}" for i in range(embeddings.shape[1])]
+        df = pd.DataFrame(embeddings, columns=feat_cols)
+        df["label"] = labels
+        df["filepath"] = image_paths
+
+        engine = cls(df, target_col="label")
+        engine._embeddings = embeddings
+        engine._input_mode = "images"
+        return engine
+
+    @classmethod
+    def from_texts(
+        cls,
+        texts: list[str],
+        labels: list[int],
+        batch_size: int = 64,
+    ) -> "DatasetEngine":
+        """Create a DatasetEngine from a list of text strings.
+
+        Encodes all texts with CLIP into feature vectors.
+
+        Parameters
+        ----------
+        texts : list of str — text strings to encode.
+        labels : list of int — integer label for each text.
+        batch_size : int — CLIP encoding batch size.
+
+        Returns
+        -------
+        DatasetEngine — ready to call analyze(), recommend(), optimize().
+        """
+        from datapruning.multimodal.clip_encoder import CLIPEncoder
+
+        if len(texts) != len(labels):
+            raise ValueError(
+                f"Number of texts ({len(texts)}) != "
+                f"number of labels ({len(labels)})"
+            )
+
+        encoder = CLIPEncoder()
+        embeddings = encoder.encode_texts(texts, batch_size=batch_size)
+
+        feat_cols = [f"clip_{i}" for i in range(embeddings.shape[1])]
+        df = pd.DataFrame(embeddings, columns=feat_cols)
+        df["label"] = labels
+        df["text"] = texts
+
+        engine = cls(df, target_col="label")
+        engine._embeddings = embeddings
+        engine._input_mode = "texts"
+        return engine
+
+    def _build_multimodal_analysis(self) -> dict:
+        """Build analysis dict from CLIP embeddings for strategy selection."""
+        X = self._embeddings
+        y = self.df[self.target_col].values
+        n_samples, n_features = X.shape
+        classes, counts = np.unique(y, return_counts=True)
+        imbalance = float(counts.max() / max(counts.min(), 1))
+
+        scan_result = {
+            "rows": n_samples,
+            "columns": n_features,
+            "memory_mb": round(X.nbytes / 1024 / 1024, 2),
+            "missing_pct": 0.0,
+            "duplicate_pct": round(
+                1.0 - len(np.unique(X, axis=0)) / n_samples, 4
+            ),
+            "numeric_pct": 100.0,
+            "categorical_pct": 0.0,
+            "estimated_training_cost": "medium",
+            "task_type": "binary" if len(classes) == 2 else "multiclass",
+            "target_balance": classes.tolist(),
+            "imbalance_ratio": imbalance,
+            "input_mode": self._input_mode,
+            "embedding_dim": n_features,
+        }
+
+        intelligence_result = {
+            "id_columns": [],
+            "date_columns": [],
+            "constant_columns": [],
+            "near_constant_columns": [],
+            "high_cardinality_columns": [],
+            "highly_correlated_pairs": [],
+            "outlier_summary": [],
+            "complexity_score": min(100, int(n_features * 0.1 + n_samples * 0.001)),
+            "leakage_risk_columns": [],
+        }
+
+        return {"scan": scan_result, "intelligence": intelligence_result}
+
+    def analyze(self) -> dict:
+        if self._embeddings is not None:
+            self._analysis = self._build_multimodal_analysis()
+        else:
+            scan_result = scan(self.df, target_col=self.target_col)
+            intelligence_result = analyze_intelligence(self.df, target_col=self.target_col)
+            self._analysis = {"scan": scan_result, "intelligence": intelligence_result}
+        return self._analysis
+
+    def report(self, as_json: bool = False):
+        if self._analysis is None:
+            self.analyze()
+        if as_json:
+            return json.dumps(self._analysis, indent=2, default=str)
+        s, i = self._analysis["scan"], self._analysis["intelligence"]
+        print(f"Rows: {s['rows']:,} | Columns: {s['columns']} | Memory: {s['memory_mb']} MB")
+        print(f"Missing: {s['missing_pct']}% | Duplicates: {s['duplicate_pct']}%")
+        if "task_type" in s:
+            print(f"Task type: {s['task_type']}")
+            if "imbalance_ratio" in s:
+                print(f"Imbalance ratio: {s['imbalance_ratio']}:1")
+        if i["leakage_risk_columns"]:
+            print(f"Possible leakage columns: {i['leakage_risk_columns']}")
+        print(f"Complexity score: {i['complexity_score']}/100")
+        return self._analysis
+
+    def recommend(self):
+        if self._analysis is None:
+            self.analyze()
+        rec = select_strategy(self._analysis)
+        if self._embeddings is not None:
+            rec._input_mode = self._input_mode
+            rec._embedding_dim = self._embeddings.shape[1]
+        return rec
+
+    def available_algorithms(self) -> list[dict]:
+        return list_available_algorithms()
+
+    def optimize(self, algorithm_key: str = None, keep_ratio: float = 0.5) -> dict:
+        if self.target_col is None:
+            raise ValueError("target_col is required to optimize.")
+
+        if self._embeddings is not None:
+            return self._optimize_multimodal(algorithm_key, keep_ratio)
+
+        if algorithm_key is None:
+            rec = self.recommend()
+            optimizer = rec.algorithm_instance
+        else:
+            optimizer = get_algorithm(algorithm_key)
+        result = run_optimization(self.df, self.target_col, optimizer, keep_ratio=keep_ratio)
+        report = build_report(optimizer, result, self.df)
+        self._last_result = result
+        self._last_optimizer = optimizer
+        return {
+            "optimized_df": result.optimized_df,
+            "explainability": report["explanation"],
+            "dataset_score": report["dataset_score"],
+            "runtime_seconds": report["runtime_seconds"],
+        }
+
+    def _optimize_multimodal(self, algorithm_key: str = None, keep_ratio: float = 0.5) -> dict:
+        """Optimize using CLIP embeddings directly."""
+        import time
+        from datapruning.research_pruning.selectors.kcenter import kcenter_greedy
+        from datapruning.research_pruning.pipeline import ResearchFusionEngine
+
+        X = self._embeddings.astype(np.float64)
+        y = self.df[self.target_col].values.astype(np.int64)
+
+        t0 = time.time()
+
+        if algorithm_key is None:
+            rec = self.recommend()
+            algorithm_key = rec.algorithm_key
+
+        n_keep = max(1, int(len(X) * keep_ratio))
+
+        if algorithm_key == "kcenter":
+            selected = kcenter_greedy(X, budget=n_keep)
+        elif algorithm_key == "fusion":
+            engine = ResearchFusionEngine({"keep_ratio": keep_ratio})
+            _, _, report = engine.run(X, y, keep_ratio=keep_ratio, auto_strategy=True)
+            selected = np.array(report.get("selected_indices", list(range(n_keep))))
+        else:
+            from datapruning.research_pruning.scores.el2n import el2n_score
+            from datapruning.research_pruning.scores.margin import margin_score
+
+            scores = np.zeros(len(X))
+            try:
+                e_scores = el2n_score(X, y)
+                scores += e_scores
+            except Exception:
+                scores += np.linalg.norm(X, axis=1)
+            try:
+                m_scores = margin_score(X, y)
+                scores += m_scores
+            except Exception:
+                pass
+
+            selected = np.argsort(scores)[::-1][:n_keep]
+
+        runtime = time.time() - t0
+        pruned_df = self.df.iloc[selected].reset_index(drop=True)
+
+        self._last_result = type("Result", (), {"optimized_df": pruned_df})()
+
+        return {
+            "optimized_df": pruned_df,
+            "explainability": {
+                "algorithm": algorithm_key,
+                "input_mode": self._input_mode,
+                "embedding_dim": self._embeddings.shape[1],
+                "rows_removed": len(self.df) - len(pruned_df),
+                "reduction_pct": round((1 - len(pruned_df) / len(self.df)) * 100, 1),
+            },
+            "dataset_score": {
+                "overall_health": 100.0,
+                "input_mode": self._input_mode,
+            },
+            "runtime_seconds": round(runtime, 4),
+        }
+
+    def export(self, path: str, result: dict = None):
+        result = result or {"optimized_df": self._last_result.optimized_df}
+        result["optimized_df"].to_csv(path, index=False)
+        print(f"Exported optimized dataset -> {path} ({len(result['optimized_df']):,} rows)")
+        return path
+
+    def benchmark(self, algorithms: list[str] = None, keep_ratio: float = 0.5) -> dict:
+        return benchmark_algorithms(self.df, self.target_col, algorithms=algorithms, keep_ratio=keep_ratio)
+
+    def compare(self, optimized_df: pd.DataFrame) -> dict:
+        return compare_to_full(optimized_df, self.df, self.target_col)

{datapruning-2.0.0 → datapruning-2.1.0/datapruning.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 2.0.0
+Version: 2.1.0
 Summary: Intelligent data pruning for ML datasets
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com
@@ -21,6 +21,9 @@ Requires-Dist: numpy>=1.24.0
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: scikit-learn>=1.3.0
 Requires-Dist: torch>=2.0.0
+Provides-Extra: clip
+Requires-Dist: transformers>=4.30.0; extra == "clip"
+Requires-Dist: pillow>=9.0.0; extra == "clip"
 Dynamic: license-file
 
 # DataPruning

{datapruning-2.0.0 → datapruning-2.1.0}/datapruning.egg-info/SOURCES.txt

@@ -28,6 +28,8 @@ datapruning/engine/optimizer.py
 datapruning/engine/scanner.py
 datapruning/engine/strategy_display.py
 datapruning/engine/strategy_selector.py
+datapruning/multimodal/__init__.py
+datapruning/multimodal/clip_encoder.py
 datapruning/reports/__init__.py
 datapruning/reports/exporter.py
 datapruning/research_pruning/__init__.py

{datapruning-2.0.0 → datapruning-2.1.0}/datapruning.egg-info/requires.txt

@@ -2,3 +2,7 @@ numpy>=1.24.0
 pandas>=2.0.0
 scikit-learn>=1.3.0
 torch>=2.0.0
+
+[clip]
+transformers>=4.30.0
+pillow>=9.0.0

{datapruning-2.0.0 → datapruning-2.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datapruning"
-version = "2.0.0"
+version = "2.1.0"
 description = "Intelligent data pruning for ML datasets"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -28,6 +28,12 @@ dependencies = [
     "torch>=2.0.0",
 ]
 
+[project.optional-dependencies]
+clip = [
+    "transformers>=4.30.0",
+    "pillow>=9.0.0",
+]
+
 [project.urls]
 Homepage = "https://www.datapruning.com"
 

datapruning-2.0.0/datapruning/sdk.py

@@ -1,80 +0,0 @@
-from __future__ import annotations
-import json
-import pandas as pd
-from datapruning.engine.scanner import scan
-from datapruning.engine.intelligence import analyze_intelligence
-from datapruning.engine.strategy_selector import select_strategy, get_algorithm, list_available_algorithms
-from datapruning.engine.optimizer import run_optimization
-from datapruning.engine.explainability import build_report
-from datapruning.engine.benchmark import benchmark_algorithms, compare_to_full
-
-
-class DatasetEngine:
-    def __init__(self, df: pd.DataFrame, target_col: str = None):
-        self.df = df
-        self.target_col = target_col
-        self._analysis = None
-        self._last_result = None
-        self._last_optimizer = None
-
-    def analyze(self) -> dict:
-        scan_result = scan(self.df, target_col=self.target_col)
-        intelligence_result = analyze_intelligence(self.df, target_col=self.target_col)
-        self._analysis = {"scan": scan_result, "intelligence": intelligence_result}
-        return self._analysis
-
-    def report(self, as_json: bool = False):
-        if self._analysis is None:
-            self.analyze()
-        if as_json:
-            return json.dumps(self._analysis, indent=2, default=str)
-        s, i = self._analysis["scan"], self._analysis["intelligence"]
-        print(f"Rows: {s['rows']:,} | Columns: {s['columns']} | Memory: {s['memory_mb']} MB")
-        print(f"Missing: {s['missing_pct']}% | Duplicates: {s['duplicate_pct']}%")
-        if "task_type" in s:
-            print(f"Task type: {s['task_type']}")
-            if "imbalance_ratio" in s:
-                print(f"Imbalance ratio: {s['imbalance_ratio']}:1")
-        if i["leakage_risk_columns"]:
-            print(f"Possible leakage columns: {i['leakage_risk_columns']}")
-        print(f"Complexity score: {i['complexity_score']}/100")
-        return self._analysis
-
-    def recommend(self):
-        if self._analysis is None:
-            self.analyze()
-        return select_strategy(self._analysis)
-
-    def available_algorithms(self) -> list[dict]:
-        return list_available_algorithms()
-
-    def optimize(self, algorithm_key: str = None, keep_ratio: float = 0.5) -> dict:
-        if self.target_col is None:
-            raise ValueError("target_col is required to optimize.")
-        if algorithm_key is None:
-            rec = self.recommend()
-            optimizer = rec.algorithm_instance
-        else:
-            optimizer = get_algorithm(algorithm_key)
-        result = run_optimization(self.df, self.target_col, optimizer, keep_ratio=keep_ratio)
-        report = build_report(optimizer, result, self.df)
-        self._last_result = result
-        self._last_optimizer = optimizer
-        return {
-            "optimized_df": result.optimized_df,
-            "explainability": report["explanation"],
-            "dataset_score": report["dataset_score"],
-            "runtime_seconds": report["runtime_seconds"],
-        }
-
-    def export(self, path: str, result: dict = None):
-        result = result or {"optimized_df": self._last_result.optimized_df}
-        result["optimized_df"].to_csv(path, index=False)
-        print(f"Exported optimized dataset -> {path} ({len(result['optimized_df']):,} rows)")
-        return path
-
-    def benchmark(self, algorithms: list[str] = None, keep_ratio: float = 0.5) -> dict:
-        return benchmark_algorithms(self.df, self.target_col, algorithms=algorithms, keep_ratio=keep_ratio)
-
-    def compare(self, optimized_df: pd.DataFrame) -> dict:
-        return compare_to_full(optimized_df, self.df, self.target_col)