edaprep 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {edaprep-0.2.1/src/edaprep.egg-info → edaprep-0.2.2}/PKG-INFO +2 -2
- {edaprep-0.2.1 → edaprep-0.2.2}/README.md +1 -1
- edaprep-0.2.2/src/edaprep/_version.py +1 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/planning/rules.py +29 -12
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/__init__.py +2 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/encoding.py +109 -9
- {edaprep-0.2.1 → edaprep-0.2.2/src/edaprep.egg-info}/PKG-INFO +2 -2
- {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_pipeline.py +69 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_transformers.py +43 -0
- edaprep-0.2.1/src/edaprep/_version.py +0 -1
- {edaprep-0.2.1 → edaprep-0.2.2}/LICENSE +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/pyproject.toml +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/setup.cfg +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/backends/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/backends/base.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/backends/pandas_backend.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/config.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/core/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/core/base.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/core/context.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/core/journal.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/core/pipeline.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/analyzer.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/categorical.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/correlation.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/numerical.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/outliers.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/target.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/exceptions.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/planning/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/planning/decisions.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/planning/planner.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/casting.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/datetime_features.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/duplicates.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/missing.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/outliers.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/scaling.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/selection.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/text.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/transformations.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/profiling/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/profiling/column_types.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/profiling/profiler.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/profiling/quality.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/profiling/statistics.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/py.typed +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/reporting/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/reporting/html.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/reporting/report.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/types.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/visualization/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/visualization/plots.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep.egg-info/SOURCES.txt +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep.egg-info/dependency_links.txt +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep.egg-info/requires.txt +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep.egg-info/top_level.txt +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_column_types.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_eda.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_leakage.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_profiler.py +0 -0
- {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_statistics.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: edaprep
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Transparent, leakage-safe EDA and ML preprocessing with an explainable planner.
|
|
5
5
|
Author-email: bijay <bijaybeezoe@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -321,7 +321,7 @@ changed what shipped are listed here alongside the commits.
|
|
|
321
321
|
| [@qiaobochi040726-source](https://github.com/qiaobochi040726-source) | Removed `Config.n_jobs`, a setting the library accepted and never read ([#11](https://github.com/bijay-odyssey/edaprep/pull/11)) |
|
|
322
322
|
| [@zbs-ops](https://github.com/zbs-ops) | Benchmarked parallelising the per-column profiling loop, which is what decided [#9](https://github.com/bijay-odyssey/edaprep/issues/9) — no commit, and the reason the parameter went |
|
|
323
323
|
| [@Jeferson681](https://github.com/Jeferson681) | Made the missing-indicator and high-missing rules see gaps introduced at the cast step ([#16](https://github.com/bijay-odyssey/edaprep/pull/16)) |
|
|
324
|
-
| [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one;
|
|
324
|
+
| [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one; that design shipped in [#18](https://github.com/bijay-odyssey/edaprep/issues/18) ([#17](https://github.com/bijay-odyssey/edaprep/pull/17)) |
|
|
325
325
|
| [@LeonxLJX](https://github.com/LeonxLJX) | Formatted the tree and made `ruff format` a real CI gate rather than an advisory one ([#25](https://github.com/bijay-odyssey/edaprep/pull/25)) |
|
|
326
326
|
| [@TrueFurina](https://github.com/TrueFurina) | First to run the Codecov upload path, which is how we learned it fails silently from a fork with no token — CI green, nothing reported; coverage moved to the job summary instead ([#21](https://github.com/bijay-odyssey/edaprep/pull/21)) |
|
|
327
327
|
|
|
@@ -275,7 +275,7 @@ changed what shipped are listed here alongside the commits.
|
|
|
275
275
|
| [@qiaobochi040726-source](https://github.com/qiaobochi040726-source) | Removed `Config.n_jobs`, a setting the library accepted and never read ([#11](https://github.com/bijay-odyssey/edaprep/pull/11)) |
|
|
276
276
|
| [@zbs-ops](https://github.com/zbs-ops) | Benchmarked parallelising the per-column profiling loop, which is what decided [#9](https://github.com/bijay-odyssey/edaprep/issues/9) — no commit, and the reason the parameter went |
|
|
277
277
|
| [@Jeferson681](https://github.com/Jeferson681) | Made the missing-indicator and high-missing rules see gaps introduced at the cast step ([#16](https://github.com/bijay-odyssey/edaprep/pull/16)) |
|
|
278
|
-
| [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one;
|
|
278
|
+
| [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one; that design shipped in [#18](https://github.com/bijay-odyssey/edaprep/issues/18) ([#17](https://github.com/bijay-odyssey/edaprep/pull/17)) |
|
|
279
279
|
| [@LeonxLJX](https://github.com/LeonxLJX) | Formatted the tree and made `ruff format` a real CI gate rather than an advisory one ([#25](https://github.com/bijay-odyssey/edaprep/pull/25)) |
|
|
280
280
|
| [@TrueFurina](https://github.com/TrueFurina) | First to run the Codecov upload path, which is how we learned it fails silently from a fork with no token — CI green, nothing reported; coverage moved to the job summary instead ([#21](https://github.com/bijay-odyssey/edaprep/pull/21)) |
|
|
281
281
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.2"
|
|
@@ -19,7 +19,7 @@ named, the reasoning is printed, and a user can override any of it per column.
|
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
21
21
|
from dataclasses import dataclass
|
|
22
|
-
from typing import Callable, Dict, List, Optional, Sequence
|
|
22
|
+
from typing import Callable, Dict, List, Optional, Sequence, Tuple
|
|
23
23
|
|
|
24
24
|
import numpy as np
|
|
25
25
|
|
|
@@ -193,9 +193,8 @@ def _rule_drop_high_missing(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
|
|
|
193
193
|
if not ctx.config.drop_high_missing or cp.is_target:
|
|
194
194
|
return None
|
|
195
195
|
threshold = ctx.thresholds.missing_drop_threshold
|
|
196
|
-
cast_missing =
|
|
197
|
-
|
|
198
|
-
if effective < threshold:
|
|
196
|
+
cast_missing, missing_fraction = _post_cast_missing(cp, ctx)
|
|
197
|
+
if missing_fraction < threshold:
|
|
199
198
|
return None
|
|
200
199
|
# Quoting cp.missing_fraction alone would report "0.0% missing" on a column that is
|
|
201
200
|
# about to be full of placeholders-turned-NaN, so the phrase names whichever
|
|
@@ -208,7 +207,7 @@ def _rule_drop_high_missing(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
|
|
|
208
207
|
elif cast_missing:
|
|
209
208
|
found = (
|
|
210
209
|
f"{cast_missing} placeholder value(s) become NaN when the column is cast, "
|
|
211
|
-
f"leaving {_pct(
|
|
210
|
+
f"leaving {_pct(missing_fraction)} missing"
|
|
212
211
|
)
|
|
213
212
|
else:
|
|
214
213
|
found = f"{_pct(cp.missing_fraction)} missing"
|
|
@@ -224,7 +223,8 @@ def _rule_drop_high_missing(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
|
|
|
224
223
|
Stage.DROP_COLUMNS,
|
|
225
224
|
"drop",
|
|
226
225
|
params={
|
|
227
|
-
"missing_fraction": round(
|
|
226
|
+
"missing_fraction": round(missing_fraction, 4),
|
|
227
|
+
"n_rows": cp.n_rows,
|
|
228
228
|
**({"cast_missing": cast_missing} if cast_missing else {}),
|
|
229
229
|
},
|
|
230
230
|
rationale=(
|
|
@@ -261,9 +261,8 @@ def _rule_missing_indicator(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
|
|
|
261
261
|
if cp.is_target or not ctx.config.add_missing_indicators:
|
|
262
262
|
return None
|
|
263
263
|
threshold = ctx.thresholds.missing_indicator_threshold
|
|
264
|
-
cast_missing =
|
|
265
|
-
|
|
266
|
-
if effective < threshold or effective >= 1.0:
|
|
264
|
+
cast_missing, missing_fraction = _post_cast_missing(cp, ctx)
|
|
265
|
+
if missing_fraction < threshold or missing_fraction >= 1.0:
|
|
267
266
|
return None
|
|
268
267
|
# Quoting cp.missing_fraction alone would report "0.0% missing" on a column that is
|
|
269
268
|
# about to be full of placeholders-turned-NaN, so the phrase names whichever
|
|
@@ -285,7 +284,8 @@ def _rule_missing_indicator(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
|
|
|
285
284
|
Stage.MISSING_FLAG,
|
|
286
285
|
"add_missing_indicator",
|
|
287
286
|
params={
|
|
288
|
-
"missing_fraction": round(
|
|
287
|
+
"missing_fraction": round(missing_fraction, 4),
|
|
288
|
+
"n_rows": cp.n_rows,
|
|
289
289
|
**({"cast_missing": cast_missing} if cast_missing else {}),
|
|
290
290
|
},
|
|
291
291
|
rationale=(
|
|
@@ -387,6 +387,22 @@ def _cast_missing(cp: ColumnProfile, ctx: RuleContext) -> int:
|
|
|
387
387
|
return sum(ctx.profile.sentinels.get(cp.name, {}).values())
|
|
388
388
|
|
|
389
389
|
|
|
390
|
+
def _post_cast_missing(cp: ColumnProfile, ctx: RuleContext) -> Tuple[int, float]:
|
|
391
|
+
"""Count and fraction of missing cells once cast-introduced NaN are counted too.
|
|
392
|
+
|
|
393
|
+
``cp.missing_fraction`` is measured on the raw frame; this is the number the
|
|
394
|
+
decision that consumes it is actually taken on. Centralising the arithmetic here
|
|
395
|
+
means the three rules keyed on it (``drop_high_missing``, ``missing_indicator``,
|
|
396
|
+
``impute_by_type``) record the same fraction in ``params`` rather than each
|
|
397
|
+
reporting the pre-cast figure while its rationale describes the post-cast one --
|
|
398
|
+
see #18. ``cp.missing_fraction`` itself is untouched and still what the rationale
|
|
399
|
+
text quotes when it deliberately contrasts "reported" against "effective".
|
|
400
|
+
"""
|
|
401
|
+
cast_missing = _cast_missing(cp, ctx)
|
|
402
|
+
missing_fraction = (cp.n_missing + cast_missing) / cp.n_rows if cp.n_rows else 0.0
|
|
403
|
+
return cast_missing, missing_fraction
|
|
404
|
+
|
|
405
|
+
|
|
390
406
|
def _rule_impute(cp: ColumnProfile, ctx: RuleContext) -> Optional[Decision]:
|
|
391
407
|
if cp.is_target:
|
|
392
408
|
return None
|
|
@@ -397,7 +413,7 @@ def _rule_impute(cp: ColumnProfile, ctx: RuleContext) -> Optional[Decision]:
|
|
|
397
413
|
outlier_may_impute = (
|
|
398
414
|
cp.semantic is SemanticType.NUMERIC and ctx.config.outlier_strategy == "impute"
|
|
399
415
|
)
|
|
400
|
-
cast_missing =
|
|
416
|
+
cast_missing, missing_fraction = _post_cast_missing(cp, ctx)
|
|
401
417
|
if cp.n_missing == 0 and not cast_missing and user is None and not outlier_may_impute:
|
|
402
418
|
return None
|
|
403
419
|
|
|
@@ -481,7 +497,8 @@ def _rule_impute(cp: ColumnProfile, ctx: RuleContext) -> Optional[Decision]:
|
|
|
481
497
|
f"impute_{strategy}",
|
|
482
498
|
params={
|
|
483
499
|
"strategy": strategy,
|
|
484
|
-
"missing_fraction": round(
|
|
500
|
+
"missing_fraction": round(missing_fraction, 4),
|
|
501
|
+
"n_rows": cp.n_rows,
|
|
485
502
|
**({"cast_missing": cast_missing} if cast_missing else {}),
|
|
486
503
|
},
|
|
487
504
|
rationale=reason,
|
|
@@ -8,6 +8,7 @@ from .casting import DataTypeInference
|
|
|
8
8
|
from .datetime_features import DateTimeExpander
|
|
9
9
|
from .duplicates import DuplicateRowHandler, duplicate_report
|
|
10
10
|
from .encoding import (
|
|
11
|
+
BinaryEncoder,
|
|
11
12
|
CategoricalEncoder,
|
|
12
13
|
FrequencyEncoder,
|
|
13
14
|
OneHotEncoder,
|
|
@@ -51,6 +52,7 @@ __all__ = [
|
|
|
51
52
|
"detect_outliers",
|
|
52
53
|
"CategoricalEncoder",
|
|
53
54
|
"OneHotEncoder",
|
|
55
|
+
"BinaryEncoder",
|
|
54
56
|
"OrdinalEncoder",
|
|
55
57
|
"FrequencyEncoder",
|
|
56
58
|
"TargetEncoder",
|
|
@@ -33,6 +33,7 @@ from ..types import CATEGORICAL_LIKE, ModelFamily, SemanticType, Severity, Stage
|
|
|
33
33
|
__all__ = [
|
|
34
34
|
"RareCategoryGrouper",
|
|
35
35
|
"OneHotEncoder",
|
|
36
|
+
"BinaryEncoder",
|
|
36
37
|
"OrdinalEncoder",
|
|
37
38
|
"FrequencyEncoder",
|
|
38
39
|
"TargetEncoder",
|
|
@@ -336,6 +337,99 @@ class OneHotEncoder(_CategoricalBase):
|
|
|
336
337
|
return out
|
|
337
338
|
|
|
338
339
|
|
|
340
|
+
class BinaryEncoder(_CategoricalBase):
|
|
341
|
+
"""Expand each category into ``ceil(log2(n))`` binary indicator columns.
|
|
342
|
+
|
|
343
|
+
A middle ground between one-hot (``n`` columns) and ordinal (one column with a
|
|
344
|
+
false ordering). Unseen categories at transform time are encoded as all-zero,
|
|
345
|
+
consistent with :class:`OneHotEncoder`.
|
|
346
|
+
"""
|
|
347
|
+
|
|
348
|
+
stage = Stage.ENCODE
|
|
349
|
+
|
|
350
|
+
def __init__(
|
|
351
|
+
self,
|
|
352
|
+
columns: Optional[Sequence[str]] = None,
|
|
353
|
+
dtype: str = "int8",
|
|
354
|
+
) -> None:
|
|
355
|
+
super().__init__(columns)
|
|
356
|
+
self.dtype = dtype
|
|
357
|
+
|
|
358
|
+
def _fit(self, X: pd.DataFrame, y: Optional[pd.Series], context: FitContext) -> None:
|
|
359
|
+
self.categories_: Dict[str, List[Any]] = {}
|
|
360
|
+
self.mappings_: Dict[str, Dict[Any, int]] = {}
|
|
361
|
+
self.n_bits_: Dict[str, int] = {}
|
|
362
|
+
self.output_names_: Dict[str, List[str]] = {}
|
|
363
|
+
|
|
364
|
+
with context.journal.timer(self.stage, type(self).__name__, "fit", "fit") as timer:
|
|
365
|
+
total = 0
|
|
366
|
+
for column in self.columns_:
|
|
367
|
+
series = self._as_object(X[column])
|
|
368
|
+
categories = sorted(series.dropna().unique(), key=_sort_key)
|
|
369
|
+
self.categories_[column] = categories
|
|
370
|
+
self.mappings_[column] = {c: i for i, c in enumerate(categories)}
|
|
371
|
+
n_bits = int(np.ceil(np.log2(max(len(categories), 1))))
|
|
372
|
+
self.n_bits_[column] = n_bits
|
|
373
|
+
self.output_names_[column] = [f"{column}__bin{i}" for i in range(n_bits)]
|
|
374
|
+
total += n_bits
|
|
375
|
+
|
|
376
|
+
timer.columns = list(self.columns_)
|
|
377
|
+
timer.effect = {"n_output_columns": total}
|
|
378
|
+
|
|
379
|
+
def _transform(self, X: pd.DataFrame, context: FitContext) -> pd.DataFrame:
|
|
380
|
+
added: Dict[str, pd.Series] = {}
|
|
381
|
+
unknown_counts: Dict[str, int] = {}
|
|
382
|
+
|
|
383
|
+
with context.journal.timer(
|
|
384
|
+
self.stage, type(self).__name__, "binary", "transform"
|
|
385
|
+
) as timer:
|
|
386
|
+
for column in self.columns_:
|
|
387
|
+
if column not in X.columns:
|
|
388
|
+
continue
|
|
389
|
+
series = self._as_object(X[column])
|
|
390
|
+
mapping = self.mappings_[column]
|
|
391
|
+
integer_codes = series.map(mapping)
|
|
392
|
+
unknown = series.notna() & integer_codes.isna()
|
|
393
|
+
n_unknown = int(unknown.sum())
|
|
394
|
+
if n_unknown:
|
|
395
|
+
unknown_counts[column] = n_unknown
|
|
396
|
+
|
|
397
|
+
known = integer_codes.notna()
|
|
398
|
+
codes_int = integer_codes[known].to_numpy(dtype=np.int64)
|
|
399
|
+
for bit, name in enumerate(self.output_names_[column]):
|
|
400
|
+
values = np.zeros(len(X), dtype=self.dtype)
|
|
401
|
+
if known.any():
|
|
402
|
+
values[known.to_numpy()] = ((codes_int >> bit) & 1).astype(self.dtype)
|
|
403
|
+
added[name] = pd.Series(values, index=X.index, name=name)
|
|
404
|
+
|
|
405
|
+
if unknown_counts:
|
|
406
|
+
context.journal.warn(
|
|
407
|
+
"unseen_categories",
|
|
408
|
+
f"{sum(unknown_counts.values())} value(s) across "
|
|
409
|
+
f"{len(unknown_counts)} column(s) were not present at fit time and "
|
|
410
|
+
f"were encoded as all-zero indicators: "
|
|
411
|
+
f"{', '.join(sorted(unknown_counts))}.",
|
|
412
|
+
Severity.INFO,
|
|
413
|
+
tuple(unknown_counts),
|
|
414
|
+
unknown_counts,
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
timer.columns = list(self.categories_)
|
|
418
|
+
timer.effect = {
|
|
419
|
+
"n_output_columns": len(added),
|
|
420
|
+
"n_unknown_values": unknown_counts,
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
remaining = {str(c): X[c] for c in X.columns if str(c) not in self.categories_}
|
|
424
|
+
return pd.DataFrame({**remaining, **added}, index=X.index, copy=False)
|
|
425
|
+
|
|
426
|
+
def _compute_feature_names_out(self) -> List[str]:
|
|
427
|
+
out = [c for c in self.feature_names_in_ if c not in self.categories_]
|
|
428
|
+
for column in self.categories_:
|
|
429
|
+
out.extend(self.output_names_[column])
|
|
430
|
+
return out
|
|
431
|
+
|
|
432
|
+
|
|
339
433
|
class OrdinalEncoder(_CategoricalBase):
|
|
340
434
|
"""Map categories to integer codes.
|
|
341
435
|
|
|
@@ -734,13 +828,11 @@ class CategoricalEncoder(_CategoricalBase):
|
|
|
734
828
|
if strategy == "target":
|
|
735
829
|
return TargetEncoder(cols)
|
|
736
830
|
if strategy == "binary":
|
|
737
|
-
|
|
738
|
-
"encoding='binary' is not implemented in this version. Use 'frequency' "
|
|
739
|
-
"or 'target' for high-cardinality columns; both produce a single "
|
|
740
|
-
"column and are better understood."
|
|
741
|
-
)
|
|
831
|
+
return BinaryEncoder(cols)
|
|
742
832
|
raise ConfigurationError.unknown_option(
|
|
743
|
-
"encoding",
|
|
833
|
+
"encoding",
|
|
834
|
+
strategy,
|
|
835
|
+
["onehot", "ordinal", "frequency", "count", "target", "binary"],
|
|
744
836
|
)
|
|
745
837
|
|
|
746
838
|
def _fit_transform(
|
|
@@ -774,12 +866,17 @@ class CategoricalEncoder(_CategoricalBase):
|
|
|
774
866
|
return X[keep]
|
|
775
867
|
|
|
776
868
|
def _compute_feature_names_out(self) -> List[str]:
|
|
777
|
-
# Must match _transform exactly. One-hot
|
|
778
|
-
# end rather than expanding in place (expanding in place would mean rebuilding
|
|
869
|
+
# Must match _transform exactly. One-hot and binary append indicator columns at
|
|
870
|
+
# the end rather than expanding in place (expanding in place would mean rebuilding
|
|
779
871
|
# the frame around each encoded column), so the names have to be appended too.
|
|
780
872
|
# Ordinal, frequency and target encoding all replace their column in position.
|
|
781
873
|
onehot = self.encoders_.get("onehot")
|
|
782
|
-
|
|
874
|
+
binary = self.encoders_.get("binary")
|
|
875
|
+
encoded_away = set()
|
|
876
|
+
if onehot is not None:
|
|
877
|
+
encoded_away.update(onehot.categories_) # type: ignore[attr-defined]
|
|
878
|
+
if binary is not None:
|
|
879
|
+
encoded_away.update(binary.categories_) # type: ignore[attr-defined]
|
|
783
880
|
names = [
|
|
784
881
|
c
|
|
785
882
|
for c in self.feature_names_in_
|
|
@@ -788,4 +885,7 @@ class CategoricalEncoder(_CategoricalBase):
|
|
|
788
885
|
if onehot is not None:
|
|
789
886
|
for column in onehot.categories_: # type: ignore[attr-defined]
|
|
790
887
|
names.extend(onehot.output_names_[column]) # type: ignore[attr-defined]
|
|
888
|
+
if binary is not None:
|
|
889
|
+
for column in binary.categories_: # type: ignore[attr-defined]
|
|
890
|
+
names.extend(binary.output_names_[column]) # type: ignore[attr-defined]
|
|
791
891
|
return names
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: edaprep
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Transparent, leakage-safe EDA and ML preprocessing with an explainable planner.
|
|
5
5
|
Author-email: bijay <bijaybeezoe@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -321,7 +321,7 @@ changed what shipped are listed here alongside the commits.
|
|
|
321
321
|
| [@qiaobochi040726-source](https://github.com/qiaobochi040726-source) | Removed `Config.n_jobs`, a setting the library accepted and never read ([#11](https://github.com/bijay-odyssey/edaprep/pull/11)) |
|
|
322
322
|
| [@zbs-ops](https://github.com/zbs-ops) | Benchmarked parallelising the per-column profiling loop, which is what decided [#9](https://github.com/bijay-odyssey/edaprep/issues/9) — no commit, and the reason the parameter went |
|
|
323
323
|
| [@Jeferson681](https://github.com/Jeferson681) | Made the missing-indicator and high-missing rules see gaps introduced at the cast step ([#16](https://github.com/bijay-odyssey/edaprep/pull/16)) |
|
|
324
|
-
| [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one;
|
|
324
|
+
| [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one; that design shipped in [#18](https://github.com/bijay-odyssey/edaprep/issues/18) ([#17](https://github.com/bijay-odyssey/edaprep/pull/17)) |
|
|
325
325
|
| [@LeonxLJX](https://github.com/LeonxLJX) | Formatted the tree and made `ruff format` a real CI gate rather than an advisory one ([#25](https://github.com/bijay-odyssey/edaprep/pull/25)) |
|
|
326
326
|
| [@TrueFurina](https://github.com/TrueFurina) | First to run the Codecov upload path, which is how we learned it fails silently from a fork with no token — CI green, nothing reported; coverage moved to the job summary instead ([#21](https://github.com/bijay-odyssey/edaprep/pull/21)) |
|
|
327
327
|
|
|
@@ -794,6 +794,75 @@ def test_column_with_real_nan_still_gets_a_missing_indicator() -> None:
|
|
|
794
794
|
assert int(out["amount__was_missing"].sum()) == 24
|
|
795
795
|
|
|
796
796
|
|
|
797
|
+
def test_missing_fraction_is_consistent_across_the_three_cast_missing_rules() -> None:
|
|
798
|
+
"""#18: drop_high_missing, missing_indicator and impute_by_type must agree.
|
|
799
|
+
|
|
800
|
+
Regression: all three rules read cast-introduced placeholders (see #16 / #14), but
|
|
801
|
+
only stored the pre-cast ``cp.missing_fraction`` in ``params`` while their rationale
|
|
802
|
+
quoted the post-cast figure -- a decision whose prose says "above the 5.0% flag
|
|
803
|
+
threshold" reporting ``missing_fraction=0.0`` in the machine-readable half. One frame
|
|
804
|
+
with three columns, one per rule, and every decision must report the same
|
|
805
|
+
(post-cast) fraction plus the ``n_rows`` needed to reconstruct it.
|
|
806
|
+
"""
|
|
807
|
+
gen = np.random.default_rng(41)
|
|
808
|
+
n = 300
|
|
809
|
+
|
|
810
|
+
def blanks(count: int) -> list:
|
|
811
|
+
values = [f"{v:.2f}" for v in gen.uniform(20, 8000, size=n)]
|
|
812
|
+
for i in range(count):
|
|
813
|
+
values[i] = ""
|
|
814
|
+
return values
|
|
815
|
+
|
|
816
|
+
frame = pd.DataFrame(
|
|
817
|
+
{
|
|
818
|
+
"col_impute": blanks(6), # 2%: below the 5% indicator threshold
|
|
819
|
+
"col_indicator": blanks(24), # 8%: indicator, then impute
|
|
820
|
+
"col_drop": blanks(210), # 70%: above the 60% drop ceiling
|
|
821
|
+
"y": gen.integers(0, 2, size=n),
|
|
822
|
+
}
|
|
823
|
+
)
|
|
824
|
+
|
|
825
|
+
pipe = AutoPipeline(target="y", model_family="linear", random_state=0)
|
|
826
|
+
pipe.fit(frame)
|
|
827
|
+
plan = pipe.plan_
|
|
828
|
+
|
|
829
|
+
expected = {
|
|
830
|
+
"col_impute": (6, 6 / n),
|
|
831
|
+
"col_indicator": (24, 24 / n),
|
|
832
|
+
"col_drop": (210, 210 / n),
|
|
833
|
+
}
|
|
834
|
+
tracked_actions = {"drop", "add_missing_indicator"}
|
|
835
|
+
for column, (cast_missing, fraction) in expected.items():
|
|
836
|
+
for decision in plan.decisions:
|
|
837
|
+
if decision.column != column:
|
|
838
|
+
continue
|
|
839
|
+
if decision.action not in tracked_actions and not decision.action.startswith(
|
|
840
|
+
"impute_"
|
|
841
|
+
):
|
|
842
|
+
continue
|
|
843
|
+
assert decision.params.get("cast_missing") == cast_missing, decision
|
|
844
|
+
assert decision.params["missing_fraction"] == pytest.approx(fraction), decision
|
|
845
|
+
assert decision.params["n_rows"] == n, decision
|
|
846
|
+
|
|
847
|
+
actions = {d.column: d.action for d in plan.decisions if d.action.startswith("impute_")}
|
|
848
|
+
assert actions.get("col_impute", "").startswith("impute_")
|
|
849
|
+
assert actions.get("col_indicator", "").startswith("impute_")
|
|
850
|
+
assert "col_drop" not in actions, (
|
|
851
|
+
"a dropped column must not also be scheduled for imputation"
|
|
852
|
+
)
|
|
853
|
+
indicator_columns = {
|
|
854
|
+
d.column for d in plan.decisions if d.action == "add_missing_indicator"
|
|
855
|
+
}
|
|
856
|
+
assert indicator_columns == {"col_indicator"}
|
|
857
|
+
assert "col_drop" in plan.dropped_columns
|
|
858
|
+
|
|
859
|
+
# The new key must survive a JSON round-trip like every other param.
|
|
860
|
+
restored = Plan.from_dict(json.loads(plan.to_json()))
|
|
861
|
+
before = {(d.column, d.action): d.params for d in plan.decisions}
|
|
862
|
+
after = {(d.column, d.action): d.params for d in restored.decisions}
|
|
863
|
+
assert before == after
|
|
864
|
+
|
|
865
|
+
|
|
797
866
|
def test_from_dict_tolerates_settings_this_version_removed() -> None:
|
|
798
867
|
"""A Config saved before a setting was retired must still load.
|
|
799
868
|
|
|
@@ -15,6 +15,7 @@ from edaprep.exceptions import (
|
|
|
15
15
|
TransformationError,
|
|
16
16
|
)
|
|
17
17
|
from edaprep.preprocessing import (
|
|
18
|
+
BinaryEncoder,
|
|
18
19
|
CategoricalEncoder,
|
|
19
20
|
ColumnDropper,
|
|
20
21
|
ConstantFilter,
|
|
@@ -396,6 +397,47 @@ def test_onehot_drop_first() -> None:
|
|
|
396
397
|
assert list(out.columns) == ["c_b", "c_c"]
|
|
397
398
|
|
|
398
399
|
|
|
400
|
+
def test_binary_encoder_distinct_categories_get_distinct_codes() -> None:
|
|
401
|
+
frame = pd.DataFrame({"c": ["a", "b", "c", "d"]})
|
|
402
|
+
out = BinaryEncoder(["c"]).fit_transform(frame, None, ctx(frame))
|
|
403
|
+
codes = [tuple(row) for row in out.to_numpy()]
|
|
404
|
+
assert len(codes) == len(set(codes))
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def test_binary_encoder_output_width() -> None:
|
|
408
|
+
for n in (1, 2, 3, 4, 5, 8, 9):
|
|
409
|
+
categories = [f"v{i}" for i in range(n)]
|
|
410
|
+
frame = pd.DataFrame({"c": categories})
|
|
411
|
+
out = BinaryEncoder(["c"]).fit_transform(frame, None, ctx(frame))
|
|
412
|
+
expected = int(np.ceil(np.log2(max(n, 1))))
|
|
413
|
+
assert len([c for c in out.columns if c.startswith("c__bin")]) == expected
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def test_binary_column_order_is_deterministic() -> None:
|
|
417
|
+
gen = np.random.default_rng(81)
|
|
418
|
+
frame = pd.DataFrame({"c": gen.choice(list("dcba"), 100)})
|
|
419
|
+
a = BinaryEncoder(["c"]).fit_transform(frame, None, ctx(frame))
|
|
420
|
+
b = BinaryEncoder(["c"]).fit_transform(frame.iloc[::-1], None, ctx(frame))
|
|
421
|
+
assert list(a.columns) == list(b.columns)
|
|
422
|
+
assert list(a.columns) == ["c__bin0", "c__bin1"]
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def test_binary_encoder_unseen_category_is_all_zero() -> None:
|
|
426
|
+
train = pd.DataFrame({"c": ["a", "b", "c", "d"] * 20})
|
|
427
|
+
context = ctx(train)
|
|
428
|
+
encoder = BinaryEncoder(["c"]).fit(train, None, context)
|
|
429
|
+
out = encoder.transform(pd.DataFrame({"c": ["zzz"]}), context)
|
|
430
|
+
assert out.to_numpy().tolist() == [[0, 0]]
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def test_binary_encoder_feature_names_match_columns() -> None:
|
|
434
|
+
frame = pd.DataFrame({"c": ["a", "b", "c", "d", "e"]})
|
|
435
|
+
context = ctx(frame)
|
|
436
|
+
encoder = BinaryEncoder(["c"]).fit(frame, None, context)
|
|
437
|
+
out = encoder.transform(frame, context)
|
|
438
|
+
assert list(encoder.get_feature_names_out()) == list(out.columns)
|
|
439
|
+
|
|
440
|
+
|
|
399
441
|
def test_ordinal_encoder_honours_an_ordered_categorical() -> None:
|
|
400
442
|
dtype = pd.CategoricalDtype(["low", "medium", "high"], ordered=True)
|
|
401
443
|
frame = pd.DataFrame({"c": pd.Series(["high", "low", "medium"], dtype=dtype)})
|
|
@@ -798,6 +840,7 @@ ALL_TRANSFORMERS = [
|
|
|
798
840
|
OutlierHandler,
|
|
799
841
|
CategoricalEncoder,
|
|
800
842
|
OneHotEncoder,
|
|
843
|
+
BinaryEncoder,
|
|
801
844
|
OrdinalEncoder,
|
|
802
845
|
FrequencyEncoder,
|
|
803
846
|
RareCategoryGrouper,
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.2.1"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|