edaprep 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {edaprep-0.2.1/src/edaprep.egg-info → edaprep-0.3.0}/PKG-INFO +2 -2
- {edaprep-0.2.1 → edaprep-0.3.0}/README.md +1 -1
- edaprep-0.3.0/src/edaprep/_version.py +1 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/config.py +44 -4
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/eda/categorical.py +2 -1
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/planning/rules.py +34 -12
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/preprocessing/__init__.py +2 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/preprocessing/casting.py +1 -1
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/preprocessing/datetime_features.py +7 -7
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/preprocessing/encoding.py +122 -10
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/preprocessing/missing.py +182 -9
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/preprocessing/text.py +9 -3
- {edaprep-0.2.1 → edaprep-0.3.0/src/edaprep.egg-info}/PKG-INFO +2 -2
- {edaprep-0.2.1 → edaprep-0.3.0}/tests/test_eda.py +26 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/tests/test_leakage.py +38 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/tests/test_pipeline.py +121 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/tests/test_transformers.py +323 -0
- edaprep-0.2.1/src/edaprep/_version.py +0 -1
- {edaprep-0.2.1 → edaprep-0.3.0}/LICENSE +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/pyproject.toml +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/setup.cfg +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/backends/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/backends/base.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/backends/pandas_backend.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/core/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/core/base.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/core/context.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/core/journal.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/core/pipeline.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/eda/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/eda/analyzer.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/eda/correlation.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/eda/numerical.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/eda/outliers.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/eda/target.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/exceptions.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/planning/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/planning/decisions.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/planning/planner.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/preprocessing/duplicates.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/preprocessing/outliers.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/preprocessing/scaling.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/preprocessing/selection.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/preprocessing/transformations.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/profiling/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/profiling/column_types.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/profiling/profiler.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/profiling/quality.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/profiling/statistics.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/py.typed +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/reporting/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/reporting/html.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/reporting/report.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/types.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/visualization/__init__.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep/visualization/plots.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep.egg-info/SOURCES.txt +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep.egg-info/dependency_links.txt +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep.egg-info/requires.txt +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/src/edaprep.egg-info/top_level.txt +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/tests/test_column_types.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/tests/test_profiler.py +0 -0
- {edaprep-0.2.1 → edaprep-0.3.0}/tests/test_statistics.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: edaprep
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Transparent, leakage-safe EDA and ML preprocessing with an explainable planner.
|
|
5
5
|
Author-email: bijay <bijaybeezoe@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -321,7 +321,7 @@ changed what shipped are listed here alongside the commits.
|
|
|
321
321
|
| [@qiaobochi040726-source](https://github.com/qiaobochi040726-source) | Removed `Config.n_jobs`, a setting the library accepted and never read ([#11](https://github.com/bijay-odyssey/edaprep/pull/11)) |
|
|
322
322
|
| [@zbs-ops](https://github.com/zbs-ops) | Benchmarked parallelising the per-column profiling loop, which is what decided [#9](https://github.com/bijay-odyssey/edaprep/issues/9) — no commit, and the reason the parameter went |
|
|
323
323
|
| [@Jeferson681](https://github.com/Jeferson681) | Made the missing-indicator and high-missing rules see gaps introduced at the cast step ([#16](https://github.com/bijay-odyssey/edaprep/pull/16)) |
|
|
324
|
-
| [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one;
|
|
324
|
+
| [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one; that design shipped in [#18](https://github.com/bijay-odyssey/edaprep/issues/18) ([#17](https://github.com/bijay-odyssey/edaprep/pull/17)) |
|
|
325
325
|
| [@LeonxLJX](https://github.com/LeonxLJX) | Formatted the tree and made `ruff format` a real CI gate rather than an advisory one ([#25](https://github.com/bijay-odyssey/edaprep/pull/25)) |
|
|
326
326
|
| [@TrueFurina](https://github.com/TrueFurina) | First to run the Codecov upload path, which is how we learned it fails silently from a fork with no token — CI green, nothing reported; coverage moved to the job summary instead ([#21](https://github.com/bijay-odyssey/edaprep/pull/21)) |
|
|
327
327
|
|
|
@@ -275,7 +275,7 @@ changed what shipped are listed here alongside the commits.
|
|
|
275
275
|
| [@qiaobochi040726-source](https://github.com/qiaobochi040726-source) | Removed `Config.n_jobs`, a setting the library accepted and never read ([#11](https://github.com/bijay-odyssey/edaprep/pull/11)) |
|
|
276
276
|
| [@zbs-ops](https://github.com/zbs-ops) | Benchmarked parallelising the per-column profiling loop, which is what decided [#9](https://github.com/bijay-odyssey/edaprep/issues/9) — no commit, and the reason the parameter went |
|
|
277
277
|
| [@Jeferson681](https://github.com/Jeferson681) | Made the missing-indicator and high-missing rules see gaps introduced at the cast step ([#16](https://github.com/bijay-odyssey/edaprep/pull/16)) |
|
|
278
|
-
| [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one;
|
|
278
|
+
| [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one; that design shipped in [#18](https://github.com/bijay-odyssey/edaprep/issues/18) ([#17](https://github.com/bijay-odyssey/edaprep/pull/17)) |
|
|
279
279
|
| [@LeonxLJX](https://github.com/LeonxLJX) | Formatted the tree and made `ruff format` a real CI gate rather than an advisory one ([#25](https://github.com/bijay-odyssey/edaprep/pull/25)) |
|
|
280
280
|
| [@TrueFurina](https://github.com/TrueFurina) | First to run the Codecov upload path, which is how we learned it fails silently from a fork with no token — CI green, nothing reported; coverage moved to the job summary instead ([#21](https://github.com/bijay-odyssey/edaprep/pull/21)) |
|
|
281
281
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.3.0"
|
|
@@ -36,7 +36,19 @@ __all__ = ["Config", "ColumnConfig", "Thresholds", "AUTO"]
|
|
|
36
36
|
AUTO = "auto"
|
|
37
37
|
|
|
38
38
|
_MISSING_STRATEGIES = frozenset(
|
|
39
|
-
{
|
|
39
|
+
{
|
|
40
|
+
AUTO,
|
|
41
|
+
"mean",
|
|
42
|
+
"median",
|
|
43
|
+
"mode",
|
|
44
|
+
"constant",
|
|
45
|
+
"ffill",
|
|
46
|
+
"bfill",
|
|
47
|
+
"drop_rows",
|
|
48
|
+
"none",
|
|
49
|
+
"knn",
|
|
50
|
+
"iterative",
|
|
51
|
+
}
|
|
40
52
|
)
|
|
41
53
|
_OUTLIER_METHODS = frozenset({AUTO, "iqr", "zscore", "modified_zscore", "percentile", "none"})
|
|
42
54
|
_OUTLIER_STRATEGIES = frozenset(
|
|
@@ -383,7 +395,12 @@ class Config:
|
|
|
383
395
|
"""Return the override record for ``name`` without creating one."""
|
|
384
396
|
return self.columns.get(name)
|
|
385
397
|
|
|
386
|
-
def set_columns(
|
|
398
|
+
def set_columns(
|
|
399
|
+
self,
|
|
400
|
+
overrides: Mapping[str, Mapping[str, Any]],
|
|
401
|
+
*,
|
|
402
|
+
warn_on_unknown: bool = False,
|
|
403
|
+
) -> "Config":
|
|
387
404
|
"""Bulk-apply overrides: ``{"age": {"imputation": "median"}, ...}``."""
|
|
388
405
|
for name, kwargs in overrides.items():
|
|
389
406
|
col = self.column(name)
|
|
@@ -392,6 +409,15 @@ class Config:
|
|
|
392
409
|
valid = [
|
|
393
410
|
f.name for f in dataclasses.fields(ColumnConfig) if f.name != "name"
|
|
394
411
|
]
|
|
412
|
+
if warn_on_unknown:
|
|
413
|
+
warnings.warn(
|
|
414
|
+
f"Config.from_dict ignoring unrecognised column({name!r}) "
|
|
415
|
+
f"setting: {key!r}. It was either retired in a later "
|
|
416
|
+
f"version of edaprep or is misspelt.",
|
|
417
|
+
UserWarning,
|
|
418
|
+
stacklevel=2,
|
|
419
|
+
)
|
|
420
|
+
continue
|
|
395
421
|
raise ConfigurationError.unknown_option(
|
|
396
422
|
f"column({name!r}) setting", key, valid
|
|
397
423
|
)
|
|
@@ -484,7 +510,21 @@ class Config:
|
|
|
484
510
|
equally likely to be a typo, and this is the only signal you would get.
|
|
485
511
|
"""
|
|
486
512
|
data = dict(data)
|
|
487
|
-
|
|
513
|
+
thresholds_raw = dict(data.pop("thresholds", {}))
|
|
514
|
+
known_thresholds = {f.name for f in dataclasses.fields(Thresholds)}
|
|
515
|
+
unknown_thresholds = sorted(set(thresholds_raw) - known_thresholds)
|
|
516
|
+
if unknown_thresholds:
|
|
517
|
+
warnings.warn(
|
|
518
|
+
f"Config.from_dict ignoring {len(unknown_thresholds)} unrecognised "
|
|
519
|
+
f"threshold setting(s): "
|
|
520
|
+
f"{', '.join(repr(k) for k in unknown_thresholds)}. They were either "
|
|
521
|
+
f"retired in a later version of edaprep or are misspelt; the rest of "
|
|
522
|
+
f"the threshold configuration was applied unchanged.",
|
|
523
|
+
UserWarning,
|
|
524
|
+
stacklevel=2,
|
|
525
|
+
)
|
|
526
|
+
thresholds_raw = {k: v for k, v in thresholds_raw.items() if k in known_thresholds}
|
|
527
|
+
thresholds = Thresholds(**thresholds_raw)
|
|
488
528
|
columns_raw = data.pop("columns", {})
|
|
489
529
|
|
|
490
530
|
known = {f.name for f in dataclasses.fields(cls)}
|
|
@@ -504,7 +544,7 @@ class Config:
|
|
|
504
544
|
for name, kwargs in columns_raw.items():
|
|
505
545
|
kwargs = dict(kwargs)
|
|
506
546
|
kwargs.pop("name", None)
|
|
507
|
-
cfg.set_columns({name: kwargs})
|
|
547
|
+
cfg.set_columns({name: kwargs}, warn_on_unknown=True)
|
|
508
548
|
return cfg
|
|
509
549
|
|
|
510
550
|
def __repr__(self) -> str:
|
|
@@ -9,6 +9,7 @@ from __future__ import annotations
|
|
|
9
9
|
|
|
10
10
|
from typing import List, Optional
|
|
11
11
|
|
|
12
|
+
import numpy as np
|
|
12
13
|
import pandas as pd
|
|
13
14
|
|
|
14
15
|
from ..config import Config
|
|
@@ -26,7 +27,7 @@ def categorical_summary(
|
|
|
26
27
|
rare_threshold = config.effective_rare_threshold
|
|
27
28
|
high_cardinality = config.effective_high_cardinality
|
|
28
29
|
n_rows = profile.n_rows
|
|
29
|
-
floor = max(1, int(rare_threshold * n_rows))
|
|
30
|
+
floor = max(1, int(np.ceil(rare_threshold * n_rows)))
|
|
30
31
|
|
|
31
32
|
rows: List[dict] = []
|
|
32
33
|
for name in profile.column_order:
|
|
@@ -19,7 +19,7 @@ named, the reasoning is printed, and a user can override any of it per column.
|
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
21
21
|
from dataclasses import dataclass
|
|
22
|
-
from typing import Callable, Dict, List, Optional, Sequence
|
|
22
|
+
from typing import Callable, Dict, List, Optional, Sequence, Tuple
|
|
23
23
|
|
|
24
24
|
import numpy as np
|
|
25
25
|
|
|
@@ -193,9 +193,8 @@ def _rule_drop_high_missing(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
|
|
|
193
193
|
if not ctx.config.drop_high_missing or cp.is_target:
|
|
194
194
|
return None
|
|
195
195
|
threshold = ctx.thresholds.missing_drop_threshold
|
|
196
|
-
cast_missing =
|
|
197
|
-
|
|
198
|
-
if effective < threshold:
|
|
196
|
+
cast_missing, missing_fraction = _post_cast_missing(cp, ctx)
|
|
197
|
+
if missing_fraction < threshold:
|
|
199
198
|
return None
|
|
200
199
|
# Quoting cp.missing_fraction alone would report "0.0% missing" on a column that is
|
|
201
200
|
# about to be full of placeholders-turned-NaN, so the phrase names whichever
|
|
@@ -208,7 +207,7 @@ def _rule_drop_high_missing(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
|
|
|
208
207
|
elif cast_missing:
|
|
209
208
|
found = (
|
|
210
209
|
f"{cast_missing} placeholder value(s) become NaN when the column is cast, "
|
|
211
|
-
f"leaving {_pct(
|
|
210
|
+
f"leaving {_pct(missing_fraction)} missing"
|
|
212
211
|
)
|
|
213
212
|
else:
|
|
214
213
|
found = f"{_pct(cp.missing_fraction)} missing"
|
|
@@ -224,7 +223,8 @@ def _rule_drop_high_missing(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
|
|
|
224
223
|
Stage.DROP_COLUMNS,
|
|
225
224
|
"drop",
|
|
226
225
|
params={
|
|
227
|
-
"missing_fraction": round(
|
|
226
|
+
"missing_fraction": round(missing_fraction, 4),
|
|
227
|
+
"n_rows": cp.n_rows,
|
|
228
228
|
**({"cast_missing": cast_missing} if cast_missing else {}),
|
|
229
229
|
},
|
|
230
230
|
rationale=(
|
|
@@ -261,9 +261,8 @@ def _rule_missing_indicator(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
|
|
|
261
261
|
if cp.is_target or not ctx.config.add_missing_indicators:
|
|
262
262
|
return None
|
|
263
263
|
threshold = ctx.thresholds.missing_indicator_threshold
|
|
264
|
-
cast_missing =
|
|
265
|
-
|
|
266
|
-
if effective < threshold or effective >= 1.0:
|
|
264
|
+
cast_missing, missing_fraction = _post_cast_missing(cp, ctx)
|
|
265
|
+
if missing_fraction < threshold or missing_fraction >= 1.0:
|
|
267
266
|
return None
|
|
268
267
|
# Quoting cp.missing_fraction alone would report "0.0% missing" on a column that is
|
|
269
268
|
# about to be full of placeholders-turned-NaN, so the phrase names whichever
|
|
@@ -285,7 +284,8 @@ def _rule_missing_indicator(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
|
|
|
285
284
|
Stage.MISSING_FLAG,
|
|
286
285
|
"add_missing_indicator",
|
|
287
286
|
params={
|
|
288
|
-
"missing_fraction": round(
|
|
287
|
+
"missing_fraction": round(missing_fraction, 4),
|
|
288
|
+
"n_rows": cp.n_rows,
|
|
289
289
|
**({"cast_missing": cast_missing} if cast_missing else {}),
|
|
290
290
|
},
|
|
291
291
|
rationale=(
|
|
@@ -387,6 +387,22 @@ def _cast_missing(cp: ColumnProfile, ctx: RuleContext) -> int:
|
|
|
387
387
|
return sum(ctx.profile.sentinels.get(cp.name, {}).values())
|
|
388
388
|
|
|
389
389
|
|
|
390
|
+
def _post_cast_missing(cp: ColumnProfile, ctx: RuleContext) -> Tuple[int, float]:
|
|
391
|
+
"""Count and fraction of missing cells once cast-introduced NaN are counted too.
|
|
392
|
+
|
|
393
|
+
``cp.missing_fraction`` is measured on the raw frame; this is the number the
|
|
394
|
+
decision that consumes it is actually taken on. Centralising the arithmetic here
|
|
395
|
+
means the three rules keyed on it (``drop_high_missing``, ``missing_indicator``,
|
|
396
|
+
``impute_by_type``) record the same fraction in ``params`` rather than each
|
|
397
|
+
reporting the pre-cast figure while its rationale describes the post-cast one --
|
|
398
|
+
see #18. ``cp.missing_fraction`` itself is untouched and still what the rationale
|
|
399
|
+
text quotes when it deliberately contrasts "reported" against "effective".
|
|
400
|
+
"""
|
|
401
|
+
cast_missing = _cast_missing(cp, ctx)
|
|
402
|
+
missing_fraction = (cp.n_missing + cast_missing) / cp.n_rows if cp.n_rows else 0.0
|
|
403
|
+
return cast_missing, missing_fraction
|
|
404
|
+
|
|
405
|
+
|
|
390
406
|
def _rule_impute(cp: ColumnProfile, ctx: RuleContext) -> Optional[Decision]:
|
|
391
407
|
if cp.is_target:
|
|
392
408
|
return None
|
|
@@ -397,7 +413,7 @@ def _rule_impute(cp: ColumnProfile, ctx: RuleContext) -> Optional[Decision]:
|
|
|
397
413
|
outlier_may_impute = (
|
|
398
414
|
cp.semantic is SemanticType.NUMERIC and ctx.config.outlier_strategy == "impute"
|
|
399
415
|
)
|
|
400
|
-
cast_missing =
|
|
416
|
+
cast_missing, missing_fraction = _post_cast_missing(cp, ctx)
|
|
401
417
|
if cp.n_missing == 0 and not cast_missing and user is None and not outlier_may_impute:
|
|
402
418
|
return None
|
|
403
419
|
|
|
@@ -436,6 +452,11 @@ def _rule_impute(cp: ColumnProfile, ctx: RuleContext) -> Optional[Decision]:
|
|
|
436
452
|
f"{cast_missing} placeholder value(s) become NaN when the column is cast, "
|
|
437
453
|
f"so it needs imputation despite reporting 0.0% missing"
|
|
438
454
|
)
|
|
455
|
+
elif outlier_may_impute:
|
|
456
|
+
found = (
|
|
457
|
+
"0.0% missing today, but outlier_strategy='impute' may introduce NaN at the "
|
|
458
|
+
"OUTLIERS stage that runs before this one"
|
|
459
|
+
)
|
|
439
460
|
else:
|
|
440
461
|
found = f"{_pct(cp.missing_fraction)} missing"
|
|
441
462
|
|
|
@@ -481,7 +502,8 @@ def _rule_impute(cp: ColumnProfile, ctx: RuleContext) -> Optional[Decision]:
|
|
|
481
502
|
f"impute_{strategy}",
|
|
482
503
|
params={
|
|
483
504
|
"strategy": strategy,
|
|
484
|
-
"missing_fraction": round(
|
|
505
|
+
"missing_fraction": round(missing_fraction, 4),
|
|
506
|
+
"n_rows": cp.n_rows,
|
|
485
507
|
**({"cast_missing": cast_missing} if cast_missing else {}),
|
|
486
508
|
},
|
|
487
509
|
rationale=reason,
|
|
@@ -8,6 +8,7 @@ from .casting import DataTypeInference
|
|
|
8
8
|
from .datetime_features import DateTimeExpander
|
|
9
9
|
from .duplicates import DuplicateRowHandler, duplicate_report
|
|
10
10
|
from .encoding import (
|
|
11
|
+
BinaryEncoder,
|
|
11
12
|
CategoricalEncoder,
|
|
12
13
|
FrequencyEncoder,
|
|
13
14
|
OneHotEncoder,
|
|
@@ -51,6 +52,7 @@ __all__ = [
|
|
|
51
52
|
"detect_outliers",
|
|
52
53
|
"CategoricalEncoder",
|
|
53
54
|
"OneHotEncoder",
|
|
55
|
+
"BinaryEncoder",
|
|
54
56
|
"OrdinalEncoder",
|
|
55
57
|
"FrequencyEncoder",
|
|
56
58
|
"TargetEncoder",
|
|
@@ -146,7 +146,7 @@ class DataTypeInference(Transformer, ColumnTransformerMixin):
|
|
|
146
146
|
stripped = series.astype(object).map(
|
|
147
147
|
lambda v: v.strip() if isinstance(v, str) else v
|
|
148
148
|
)
|
|
149
|
-
counts["stripped"] = int((stripped != series).sum())
|
|
149
|
+
counts["stripped"] = int(((stripped != series) & series.notna()).sum())
|
|
150
150
|
series = stripped
|
|
151
151
|
|
|
152
152
|
if "sentinels_to_nan" in actions:
|
|
@@ -43,13 +43,13 @@ _EXTRACTORS: Dict[str, Callable[[pd.Series], pd.Series]] = {
|
|
|
43
43
|
"hour": lambda s: s.dt.hour,
|
|
44
44
|
"minute": lambda s: s.dt.minute,
|
|
45
45
|
"second": lambda s: s.dt.second,
|
|
46
|
-
"is_weekend": lambda s: (s.dt.dayofweek >= 5).astype("float64"),
|
|
47
|
-
"is_month_start": lambda s: s.dt.is_month_start.astype("float64"),
|
|
48
|
-
"is_month_end": lambda s: s.dt.is_month_end.astype("float64"),
|
|
49
|
-
"is_quarter_start": lambda s: s.dt.is_quarter_start.astype("float64"),
|
|
50
|
-
"is_quarter_end": lambda s: s.dt.is_quarter_end.astype("float64"),
|
|
51
|
-
"is_year_start": lambda s: s.dt.is_year_start.astype("float64"),
|
|
52
|
-
"is_year_end": lambda s: s.dt.is_year_end.astype("float64"),
|
|
46
|
+
"is_weekend": lambda s: (s.dt.dayofweek >= 5).astype("float64").where(s.notna()),
|
|
47
|
+
"is_month_start": lambda s: s.dt.is_month_start.astype("float64").where(s.notna()),
|
|
48
|
+
"is_month_end": lambda s: s.dt.is_month_end.astype("float64").where(s.notna()),
|
|
49
|
+
"is_quarter_start": lambda s: s.dt.is_quarter_start.astype("float64").where(s.notna()),
|
|
50
|
+
"is_quarter_end": lambda s: s.dt.is_quarter_end.astype("float64").where(s.notna()),
|
|
51
|
+
"is_year_start": lambda s: s.dt.is_year_start.astype("float64").where(s.notna()),
|
|
52
|
+
"is_year_end": lambda s: s.dt.is_year_end.astype("float64").where(s.notna()),
|
|
53
53
|
"month_sin": lambda s: np.sin(2 * np.pi * s.dt.month / 12.0),
|
|
54
54
|
"month_cos": lambda s: np.cos(2 * np.pi * s.dt.month / 12.0),
|
|
55
55
|
"dayofweek_sin": lambda s: np.sin(2 * np.pi * s.dt.dayofweek / 7.0),
|
|
@@ -33,6 +33,7 @@ from ..types import CATEGORICAL_LIKE, ModelFamily, SemanticType, Severity, Stage
|
|
|
33
33
|
__all__ = [
|
|
34
34
|
"RareCategoryGrouper",
|
|
35
35
|
"OneHotEncoder",
|
|
36
|
+
"BinaryEncoder",
|
|
36
37
|
"OrdinalEncoder",
|
|
37
38
|
"FrequencyEncoder",
|
|
38
39
|
"TargetEncoder",
|
|
@@ -336,6 +337,99 @@ class OneHotEncoder(_CategoricalBase):
|
|
|
336
337
|
return out
|
|
337
338
|
|
|
338
339
|
|
|
340
|
+
class BinaryEncoder(_CategoricalBase):
|
|
341
|
+
"""Expand each category into ``ceil(log2(n))`` binary indicator columns.
|
|
342
|
+
|
|
343
|
+
A middle ground between one-hot (``n`` columns) and ordinal (one column with a
|
|
344
|
+
false ordering). Unseen categories at transform time are encoded as all-zero,
|
|
345
|
+
consistent with :class:`OneHotEncoder`.
|
|
346
|
+
"""
|
|
347
|
+
|
|
348
|
+
stage = Stage.ENCODE
|
|
349
|
+
|
|
350
|
+
def __init__(
|
|
351
|
+
self,
|
|
352
|
+
columns: Optional[Sequence[str]] = None,
|
|
353
|
+
dtype: str = "int8",
|
|
354
|
+
) -> None:
|
|
355
|
+
super().__init__(columns)
|
|
356
|
+
self.dtype = dtype
|
|
357
|
+
|
|
358
|
+
def _fit(self, X: pd.DataFrame, y: Optional[pd.Series], context: FitContext) -> None:
|
|
359
|
+
self.categories_: Dict[str, List[Any]] = {}
|
|
360
|
+
self.mappings_: Dict[str, Dict[Any, int]] = {}
|
|
361
|
+
self.n_bits_: Dict[str, int] = {}
|
|
362
|
+
self.output_names_: Dict[str, List[str]] = {}
|
|
363
|
+
|
|
364
|
+
with context.journal.timer(self.stage, type(self).__name__, "fit", "fit") as timer:
|
|
365
|
+
total = 0
|
|
366
|
+
for column in self.columns_:
|
|
367
|
+
series = self._as_object(X[column])
|
|
368
|
+
categories = sorted(series.dropna().unique(), key=_sort_key)
|
|
369
|
+
self.categories_[column] = categories
|
|
370
|
+
self.mappings_[column] = {c: i for i, c in enumerate(categories)}
|
|
371
|
+
n_bits = int(np.ceil(np.log2(max(len(categories), 1))))
|
|
372
|
+
self.n_bits_[column] = n_bits
|
|
373
|
+
self.output_names_[column] = [f"{column}__bin{i}" for i in range(n_bits)]
|
|
374
|
+
total += n_bits
|
|
375
|
+
|
|
376
|
+
timer.columns = list(self.columns_)
|
|
377
|
+
timer.effect = {"n_output_columns": total}
|
|
378
|
+
|
|
379
|
+
def _transform(self, X: pd.DataFrame, context: FitContext) -> pd.DataFrame:
|
|
380
|
+
added: Dict[str, pd.Series] = {}
|
|
381
|
+
unknown_counts: Dict[str, int] = {}
|
|
382
|
+
|
|
383
|
+
with context.journal.timer(
|
|
384
|
+
self.stage, type(self).__name__, "binary", "transform"
|
|
385
|
+
) as timer:
|
|
386
|
+
for column in self.columns_:
|
|
387
|
+
if column not in X.columns:
|
|
388
|
+
continue
|
|
389
|
+
series = self._as_object(X[column])
|
|
390
|
+
mapping = self.mappings_[column]
|
|
391
|
+
integer_codes = series.map(mapping)
|
|
392
|
+
unknown = series.notna() & integer_codes.isna()
|
|
393
|
+
n_unknown = int(unknown.sum())
|
|
394
|
+
if n_unknown:
|
|
395
|
+
unknown_counts[column] = n_unknown
|
|
396
|
+
|
|
397
|
+
known = integer_codes.notna()
|
|
398
|
+
codes_int = integer_codes[known].to_numpy(dtype=np.int64)
|
|
399
|
+
for bit, name in enumerate(self.output_names_[column]):
|
|
400
|
+
values = np.zeros(len(X), dtype=self.dtype)
|
|
401
|
+
if known.any():
|
|
402
|
+
values[known.to_numpy()] = ((codes_int >> bit) & 1).astype(self.dtype)
|
|
403
|
+
added[name] = pd.Series(values, index=X.index, name=name)
|
|
404
|
+
|
|
405
|
+
if unknown_counts:
|
|
406
|
+
context.journal.warn(
|
|
407
|
+
"unseen_categories",
|
|
408
|
+
f"{sum(unknown_counts.values())} value(s) across "
|
|
409
|
+
f"{len(unknown_counts)} column(s) were not present at fit time and "
|
|
410
|
+
f"were encoded as all-zero indicators: "
|
|
411
|
+
f"{', '.join(sorted(unknown_counts))}.",
|
|
412
|
+
Severity.INFO,
|
|
413
|
+
tuple(unknown_counts),
|
|
414
|
+
unknown_counts,
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
timer.columns = list(self.categories_)
|
|
418
|
+
timer.effect = {
|
|
419
|
+
"n_output_columns": len(added),
|
|
420
|
+
"n_unknown_values": unknown_counts,
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
remaining = {str(c): X[c] for c in X.columns if str(c) not in self.categories_}
|
|
424
|
+
return pd.DataFrame({**remaining, **added}, index=X.index, copy=False)
|
|
425
|
+
|
|
426
|
+
def _compute_feature_names_out(self) -> List[str]:
|
|
427
|
+
out = [c for c in self.feature_names_in_ if c not in self.categories_]
|
|
428
|
+
for column in self.categories_:
|
|
429
|
+
out.extend(self.output_names_[column])
|
|
430
|
+
return out
|
|
431
|
+
|
|
432
|
+
|
|
339
433
|
class OrdinalEncoder(_CategoricalBase):
|
|
340
434
|
"""Map categories to integer codes.
|
|
341
435
|
|
|
@@ -343,6 +437,12 @@ class OrdinalEncoder(_CategoricalBase):
|
|
|
343
437
|
subset of codes, and for genuinely ordered columns. It imposes a false ordering on
|
|
344
438
|
a nominal column fed to a linear or distance-based model, which is why the planner
|
|
345
439
|
only selects it for ``model_family="tree"`` or ``SemanticType.ORDINAL``.
|
|
440
|
+
|
|
441
|
+
The ``dtype`` parameter (default ``"int32"``) is applied to encoded columns. When
|
|
442
|
+
missing values are present and ``dtype`` cannot represent NA (such as standard numpy
|
|
443
|
+
integer types), the column safely falls back to ``"float64"`` so downstream imputers
|
|
444
|
+
can process it. Nullable integer dtypes (e.g. ``"Int32"``) are preserved with missing
|
|
445
|
+
values.
|
|
346
446
|
"""
|
|
347
447
|
|
|
348
448
|
stage = Stage.ENCODE
|
|
@@ -403,7 +503,13 @@ class OrdinalEncoder(_CategoricalBase):
|
|
|
403
503
|
# NaN in the input stays NaN so a later imputer can see it; only
|
|
404
504
|
# genuinely unseen categories get the sentinel code.
|
|
405
505
|
codes = codes.mask(unknown, self.unknown_value)
|
|
406
|
-
|
|
506
|
+
if codes.isna().any():
|
|
507
|
+
try:
|
|
508
|
+
replacements[column] = codes.astype(self.dtype)
|
|
509
|
+
except (ValueError, TypeError):
|
|
510
|
+
replacements[column] = codes.astype("float64")
|
|
511
|
+
else:
|
|
512
|
+
replacements[column] = codes.astype(self.dtype)
|
|
407
513
|
|
|
408
514
|
if unknown_counts:
|
|
409
515
|
context.journal.warn(
|
|
@@ -734,13 +840,11 @@ class CategoricalEncoder(_CategoricalBase):
|
|
|
734
840
|
if strategy == "target":
|
|
735
841
|
return TargetEncoder(cols)
|
|
736
842
|
if strategy == "binary":
|
|
737
|
-
|
|
738
|
-
"encoding='binary' is not implemented in this version. Use 'frequency' "
|
|
739
|
-
"or 'target' for high-cardinality columns; both produce a single "
|
|
740
|
-
"column and are better understood."
|
|
741
|
-
)
|
|
843
|
+
return BinaryEncoder(cols)
|
|
742
844
|
raise ConfigurationError.unknown_option(
|
|
743
|
-
"encoding",
|
|
845
|
+
"encoding",
|
|
846
|
+
strategy,
|
|
847
|
+
["onehot", "ordinal", "frequency", "count", "target", "binary"],
|
|
744
848
|
)
|
|
745
849
|
|
|
746
850
|
def _fit_transform(
|
|
@@ -774,12 +878,17 @@ class CategoricalEncoder(_CategoricalBase):
|
|
|
774
878
|
return X[keep]
|
|
775
879
|
|
|
776
880
|
def _compute_feature_names_out(self) -> List[str]:
|
|
777
|
-
# Must match _transform exactly. One-hot
|
|
778
|
-
# end rather than expanding in place (expanding in place would mean rebuilding
|
|
881
|
+
# Must match _transform exactly. One-hot and binary append indicator columns at
|
|
882
|
+
# the end rather than expanding in place (expanding in place would mean rebuilding
|
|
779
883
|
# the frame around each encoded column), so the names have to be appended too.
|
|
780
884
|
# Ordinal, frequency and target encoding all replace their column in position.
|
|
781
885
|
onehot = self.encoders_.get("onehot")
|
|
782
|
-
|
|
886
|
+
binary = self.encoders_.get("binary")
|
|
887
|
+
encoded_away = set()
|
|
888
|
+
if onehot is not None:
|
|
889
|
+
encoded_away.update(onehot.categories_) # type: ignore[attr-defined]
|
|
890
|
+
if binary is not None:
|
|
891
|
+
encoded_away.update(binary.categories_) # type: ignore[attr-defined]
|
|
783
892
|
names = [
|
|
784
893
|
c
|
|
785
894
|
for c in self.feature_names_in_
|
|
@@ -788,4 +897,7 @@ class CategoricalEncoder(_CategoricalBase):
|
|
|
788
897
|
if onehot is not None:
|
|
789
898
|
for column in onehot.categories_: # type: ignore[attr-defined]
|
|
790
899
|
names.extend(onehot.output_names_[column]) # type: ignore[attr-defined]
|
|
900
|
+
if binary is not None:
|
|
901
|
+
for column in binary.categories_: # type: ignore[attr-defined]
|
|
902
|
+
names.extend(binary.output_names_[column]) # type: ignore[attr-defined]
|
|
791
903
|
return names
|