datapruning 1.0.9__tar.gz → 1.0.10__tar.gz

{datapruning-1.0.9/datapruning.egg-info → datapruning-1.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 1.0.9
+Version: 1.0.10
 Summary: Intelligent data pruning for ML datasets
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com

datapruning-1.0.10/datapruning/engine/explainability.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+import pandas as pd
+from datapruning.algorithms.base import BaseOptimizer, OptimizationResult, Explanation
+
+
+def _clamp(v: float, lo: float = 0.0, hi: float = 100.0) -> float:
+    return max(lo, min(hi, v))
+
+
+def build_report(optimizer: BaseOptimizer, result: OptimizationResult,
+                  df_before: pd.DataFrame) -> dict:
+    explanation: Explanation = optimizer.explain(result)
+
+    missing_pct_before = (df_before.isna().sum().sum() / df_before.size) * 100 if df_before.size else 0
+    dup_pct_before = (df_before.duplicated().sum() / len(df_before)) * 100 if len(df_before) else 0
+    missing_pct_after = (result.optimized_df.isna().sum().sum() / result.optimized_df.size) * 100 \
+        if result.optimized_df.size else 0
+    dup_pct_after = (result.optimized_df.duplicated().sum() / len(result.optimized_df)) * 100 \
+        if len(result.optimized_df) else 0
+
+    n_cols = len(df_before.columns)
+    constant_ratio = 0
+    near_constant_ratio = 0
+    outlier_ratio = 0
+    imbalance_penalty = 0
+    correlation_penalty = 0
+
+    if n_cols > 1 and hasattr(df_before, "describe"):
+        const_count = sum(1 for c in df_before.columns if df_before[c].nunique(dropna=False) <= 1)
+        constant_ratio = const_count / n_cols
+
+    for col in df_before.select_dtypes(include="number"):
+        q1, q3 = df_before[col].quantile(0.25), df_before[col].quantile(0.75)
+        iqr = q3 - q1
+        if iqr > 0:
+            outliers = ((df_before[col] < q1 - 1.5 * iqr) | (df_before[col] > q3 + 1.5 * iqr)).sum()
+            outlier_ratio = max(outlier_ratio, outliers / len(df_before))
+
+    imbalance_penalty = dup_pct_before / 100
+
+    quality = _clamp(100 - missing_pct_before * 2 - dup_pct_before * 2 - constant_ratio * 30)
+    completeness = _clamp(100 - missing_pct_before * 3)
+    balance = _clamp(100 - imbalance_penalty * 50)
+    redundancy = _clamp(100 - dup_pct_before * 3)
+    complexity = _clamp(100 - outlier_ratio * 40 - constant_ratio * 20)
+
+    size_reduction = float(1 - (result.rows_after / result.rows_before)) if result.rows_before else 0.0
+    training_readiness = _clamp(completeness * 0.4 + balance * 0.3 + redundancy * 0.3)
+    optimization_potential = _clamp(size_reduction * 100 + imbalance_penalty * 30 + dup_pct_before * 0.5)
+    overall_health = round((quality + completeness + balance + redundancy + complexity) / 5, 1)
+
+    return {
+        "explanation": explanation.__dict__,
+        "dataset_score": {
+            "overall_health": overall_health,
+            "quality": round(quality, 1),
+            "completeness": round(completeness, 1),
+            "balance": round(balance, 1),
+            "redundancy": round(redundancy, 1),
+            "training_readiness": round(training_readiness, 1),
+            "optimization_potential": round(optimization_potential, 1),
+        },
+        "runtime_seconds": round(result.runtime_seconds, 3),
+    }

{datapruning-1.0.9 → datapruning-1.0.10/datapruning.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapruning
-Version: 1.0.9
+Version: 1.0.10
 Summary: Intelligent data pruning for ML datasets
 License: Proprietary
 Project-URL: Homepage, https://www.datapruning.com

{datapruning-1.0.9 → datapruning-1.0.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datapruning"
-version = "1.0.9"
+version = "1.0.10"
 description = "Intelligent data pruning for ML datasets"
 readme = "README.md"
 requires-python = ">=3.10"

datapruning-1.0.9/datapruning/engine/explainability.py

@@ -1,26 +0,0 @@
-from __future__ import annotations
-import pandas as pd
-from datapruning.algorithms.base import BaseOptimizer, OptimizationResult, Explanation
-
-
-def build_report(optimizer: BaseOptimizer, result: OptimizationResult,
-                  df_before: pd.DataFrame) -> dict:
-    explanation: Explanation = optimizer.explain(result)
-    missing_pct_after = (result.optimized_df.isna().sum().sum() / result.optimized_df.size) * 100 \
-        if result.optimized_df.size else 0
-    dup_pct_after = (result.optimized_df.duplicated().sum() / len(result.optimized_df)) * 100 \
-        if len(result.optimized_df) else 0
-    quality = float(max(0, 100 - missing_pct_after * 2 - dup_pct_after * 2))
-    size_reduction = float(1 - (result.rows_after / result.rows_before)) if result.rows_before else 0.0
-    training_readiness = float(min(100, 70 + size_reduction * 60))
-    overall_health = round((quality + training_readiness) / 2, 1)
-    return {
-        "explanation": explanation.__dict__,
-        "dataset_score": {
-            "overall_health": overall_health,
-            "quality": round(quality, 1),
-            "training_readiness": round(training_readiness, 1),
-            "optimization_potential": round(size_reduction * 100, 1),
-        },
-        "runtime_seconds": round(result.runtime_seconds, 3),
-    }