edaprep 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. {edaprep-0.2.1/src/edaprep.egg-info → edaprep-0.2.2}/PKG-INFO +2 -2
  2. {edaprep-0.2.1 → edaprep-0.2.2}/README.md +1 -1
  3. edaprep-0.2.2/src/edaprep/_version.py +1 -0
  4. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/planning/rules.py +29 -12
  5. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/__init__.py +2 -0
  6. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/encoding.py +109 -9
  7. {edaprep-0.2.1 → edaprep-0.2.2/src/edaprep.egg-info}/PKG-INFO +2 -2
  8. {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_pipeline.py +69 -0
  9. {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_transformers.py +43 -0
  10. edaprep-0.2.1/src/edaprep/_version.py +0 -1
  11. {edaprep-0.2.1 → edaprep-0.2.2}/LICENSE +0 -0
  12. {edaprep-0.2.1 → edaprep-0.2.2}/pyproject.toml +0 -0
  13. {edaprep-0.2.1 → edaprep-0.2.2}/setup.cfg +0 -0
  14. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/__init__.py +0 -0
  15. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/backends/__init__.py +0 -0
  16. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/backends/base.py +0 -0
  17. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/backends/pandas_backend.py +0 -0
  18. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/config.py +0 -0
  19. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/core/__init__.py +0 -0
  20. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/core/base.py +0 -0
  21. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/core/context.py +0 -0
  22. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/core/journal.py +0 -0
  23. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/core/pipeline.py +0 -0
  24. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/__init__.py +0 -0
  25. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/analyzer.py +0 -0
  26. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/categorical.py +0 -0
  27. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/correlation.py +0 -0
  28. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/numerical.py +0 -0
  29. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/outliers.py +0 -0
  30. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/eda/target.py +0 -0
  31. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/exceptions.py +0 -0
  32. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/planning/__init__.py +0 -0
  33. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/planning/decisions.py +0 -0
  34. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/planning/planner.py +0 -0
  35. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/casting.py +0 -0
  36. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/datetime_features.py +0 -0
  37. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/duplicates.py +0 -0
  38. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/missing.py +0 -0
  39. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/outliers.py +0 -0
  40. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/scaling.py +0 -0
  41. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/selection.py +0 -0
  42. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/text.py +0 -0
  43. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/preprocessing/transformations.py +0 -0
  44. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/profiling/__init__.py +0 -0
  45. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/profiling/column_types.py +0 -0
  46. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/profiling/profiler.py +0 -0
  47. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/profiling/quality.py +0 -0
  48. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/profiling/statistics.py +0 -0
  49. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/py.typed +0 -0
  50. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/reporting/__init__.py +0 -0
  51. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/reporting/html.py +0 -0
  52. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/reporting/report.py +0 -0
  53. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/types.py +0 -0
  54. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/visualization/__init__.py +0 -0
  55. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep/visualization/plots.py +0 -0
  56. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep.egg-info/SOURCES.txt +0 -0
  57. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep.egg-info/dependency_links.txt +0 -0
  58. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep.egg-info/requires.txt +0 -0
  59. {edaprep-0.2.1 → edaprep-0.2.2}/src/edaprep.egg-info/top_level.txt +0 -0
  60. {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_column_types.py +0 -0
  61. {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_eda.py +0 -0
  62. {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_leakage.py +0 -0
  63. {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_profiler.py +0 -0
  64. {edaprep-0.2.1 → edaprep-0.2.2}/tests/test_statistics.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: edaprep
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Transparent, leakage-safe EDA and ML preprocessing with an explainable planner.
5
5
  Author-email: bijay <bijaybeezoe@gmail.com>
6
6
  License: MIT
@@ -321,7 +321,7 @@ changed what shipped are listed here alongside the commits.
321
321
  | [@qiaobochi040726-source](https://github.com/qiaobochi040726-source) | Removed `Config.n_jobs`, a setting the library accepted and never read ([#11](https://github.com/bijay-odyssey/edaprep/pull/11)) |
322
322
  | [@zbs-ops](https://github.com/zbs-ops) | Benchmarked parallelising the per-column profiling loop, which is what decided [#9](https://github.com/bijay-odyssey/edaprep/issues/9) — no commit, and the reason the parameter went |
323
323
  | [@Jeferson681](https://github.com/Jeferson681) | Made the missing-indicator and high-missing rules see gaps introduced at the cast step ([#16](https://github.com/bijay-odyssey/edaprep/pull/16)) |
324
- | [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one; it survives as [#18](https://github.com/bijay-odyssey/edaprep/issues/18) ([#17](https://github.com/bijay-odyssey/edaprep/pull/17)) |
324
+ | [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one; that design shipped in [#18](https://github.com/bijay-odyssey/edaprep/issues/18) ([#17](https://github.com/bijay-odyssey/edaprep/pull/17)) |
325
325
  | [@LeonxLJX](https://github.com/LeonxLJX) | Formatted the tree and made `ruff format` a real CI gate rather than an advisory one ([#25](https://github.com/bijay-odyssey/edaprep/pull/25)) |
326
326
  | [@TrueFurina](https://github.com/TrueFurina) | First to run the Codecov upload path, which is how we learned it fails silently from a fork with no token — CI green, nothing reported; coverage moved to the job summary instead ([#21](https://github.com/bijay-odyssey/edaprep/pull/21)) |
327
327
 
@@ -275,7 +275,7 @@ changed what shipped are listed here alongside the commits.
275
275
  | [@qiaobochi040726-source](https://github.com/qiaobochi040726-source) | Removed `Config.n_jobs`, a setting the library accepted and never read ([#11](https://github.com/bijay-odyssey/edaprep/pull/11)) |
276
276
  | [@zbs-ops](https://github.com/zbs-ops) | Benchmarked parallelising the per-column profiling loop, which is what decided [#9](https://github.com/bijay-odyssey/edaprep/issues/9) — no commit, and the reason the parameter went |
277
277
  | [@Jeferson681](https://github.com/Jeferson681) | Made the missing-indicator and high-missing rules see gaps introduced at the cast step ([#16](https://github.com/bijay-odyssey/edaprep/pull/16)) |
278
- | [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one; it survives as [#18](https://github.com/bijay-odyssey/edaprep/issues/18) ([#17](https://github.com/bijay-odyssey/edaprep/pull/17)) |
278
+ | [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one; that design shipped in [#18](https://github.com/bijay-odyssey/edaprep/issues/18) ([#17](https://github.com/bijay-odyssey/edaprep/pull/17)) |
279
279
  | [@LeonxLJX](https://github.com/LeonxLJX) | Formatted the tree and made `ruff format` a real CI gate rather than an advisory one ([#25](https://github.com/bijay-odyssey/edaprep/pull/25)) |
280
280
  | [@TrueFurina](https://github.com/TrueFurina) | First to run the Codecov upload path, which is how we learned it fails silently from a fork with no token — CI green, nothing reported; coverage moved to the job summary instead ([#21](https://github.com/bijay-odyssey/edaprep/pull/21)) |
281
281
 
@@ -0,0 +1 @@
1
+ __version__ = "0.2.2"
@@ -19,7 +19,7 @@ named, the reasoning is printed, and a user can override any of it per column.
19
19
  from __future__ import annotations
20
20
 
21
21
  from dataclasses import dataclass
22
- from typing import Callable, Dict, List, Optional, Sequence
22
+ from typing import Callable, Dict, List, Optional, Sequence, Tuple
23
23
 
24
24
  import numpy as np
25
25
 
@@ -193,9 +193,8 @@ def _rule_drop_high_missing(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
193
193
  if not ctx.config.drop_high_missing or cp.is_target:
194
194
  return None
195
195
  threshold = ctx.thresholds.missing_drop_threshold
196
- cast_missing = _cast_missing(cp, ctx)
197
- effective = (cp.n_missing + cast_missing) / cp.n_rows if cp.n_rows else 0.0
198
- if effective < threshold:
196
+ cast_missing, missing_fraction = _post_cast_missing(cp, ctx)
197
+ if missing_fraction < threshold:
199
198
  return None
200
199
  # Quoting cp.missing_fraction alone would report "0.0% missing" on a column that is
201
200
  # about to be full of placeholders-turned-NaN, so the phrase names whichever
@@ -208,7 +207,7 @@ def _rule_drop_high_missing(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
208
207
  elif cast_missing:
209
208
  found = (
210
209
  f"{cast_missing} placeholder value(s) become NaN when the column is cast, "
211
- f"leaving {_pct(effective)} missing"
210
+ f"leaving {_pct(missing_fraction)} missing"
212
211
  )
213
212
  else:
214
213
  found = f"{_pct(cp.missing_fraction)} missing"
@@ -224,7 +223,8 @@ def _rule_drop_high_missing(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
224
223
  Stage.DROP_COLUMNS,
225
224
  "drop",
226
225
  params={
227
- "missing_fraction": round(cp.missing_fraction, 4),
226
+ "missing_fraction": round(missing_fraction, 4),
227
+ "n_rows": cp.n_rows,
228
228
  **({"cast_missing": cast_missing} if cast_missing else {}),
229
229
  },
230
230
  rationale=(
@@ -261,9 +261,8 @@ def _rule_missing_indicator(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
261
261
  if cp.is_target or not ctx.config.add_missing_indicators:
262
262
  return None
263
263
  threshold = ctx.thresholds.missing_indicator_threshold
264
- cast_missing = _cast_missing(cp, ctx)
265
- effective = (cp.n_missing + cast_missing) / cp.n_rows if cp.n_rows else 0.0
266
- if effective < threshold or effective >= 1.0:
264
+ cast_missing, missing_fraction = _post_cast_missing(cp, ctx)
265
+ if missing_fraction < threshold or missing_fraction >= 1.0:
267
266
  return None
268
267
  # Quoting cp.missing_fraction alone would report "0.0% missing" on a column that is
269
268
  # about to be full of placeholders-turned-NaN, so the phrase names whichever
@@ -285,7 +284,8 @@ def _rule_missing_indicator(cp: ColumnProfile, ctx: RuleContext) -> Optional[Dec
285
284
  Stage.MISSING_FLAG,
286
285
  "add_missing_indicator",
287
286
  params={
288
- "missing_fraction": round(cp.missing_fraction, 4),
287
+ "missing_fraction": round(missing_fraction, 4),
288
+ "n_rows": cp.n_rows,
289
289
  **({"cast_missing": cast_missing} if cast_missing else {}),
290
290
  },
291
291
  rationale=(
@@ -387,6 +387,22 @@ def _cast_missing(cp: ColumnProfile, ctx: RuleContext) -> int:
387
387
  return sum(ctx.profile.sentinels.get(cp.name, {}).values())
388
388
 
389
389
 
390
+ def _post_cast_missing(cp: ColumnProfile, ctx: RuleContext) -> Tuple[int, float]:
391
+ """Count and fraction of missing cells once cast-introduced NaN are counted too.
392
+
393
+ ``cp.missing_fraction`` is measured on the raw frame; this is the number the
394
+ decision that consumes it is actually taken on. Centralising the arithmetic here
395
+ means the three rules keyed on it (``drop_high_missing``, ``missing_indicator``,
396
+ ``impute_by_type``) record the same fraction in ``params`` rather than each
397
+ reporting the pre-cast figure while its rationale describes the post-cast one --
398
+ see #18. ``cp.missing_fraction`` itself is untouched and still what the rationale
399
+ text quotes when it deliberately contrasts "reported" against "effective".
400
+ """
401
+ cast_missing = _cast_missing(cp, ctx)
402
+ missing_fraction = (cp.n_missing + cast_missing) / cp.n_rows if cp.n_rows else 0.0
403
+ return cast_missing, missing_fraction
404
+
405
+
390
406
  def _rule_impute(cp: ColumnProfile, ctx: RuleContext) -> Optional[Decision]:
391
407
  if cp.is_target:
392
408
  return None
@@ -397,7 +413,7 @@ def _rule_impute(cp: ColumnProfile, ctx: RuleContext) -> Optional[Decision]:
397
413
  outlier_may_impute = (
398
414
  cp.semantic is SemanticType.NUMERIC and ctx.config.outlier_strategy == "impute"
399
415
  )
400
- cast_missing = _cast_missing(cp, ctx)
416
+ cast_missing, missing_fraction = _post_cast_missing(cp, ctx)
401
417
  if cp.n_missing == 0 and not cast_missing and user is None and not outlier_may_impute:
402
418
  return None
403
419
 
@@ -481,7 +497,8 @@ def _rule_impute(cp: ColumnProfile, ctx: RuleContext) -> Optional[Decision]:
481
497
  f"impute_{strategy}",
482
498
  params={
483
499
  "strategy": strategy,
484
- "missing_fraction": round(cp.missing_fraction, 4),
500
+ "missing_fraction": round(missing_fraction, 4),
501
+ "n_rows": cp.n_rows,
485
502
  **({"cast_missing": cast_missing} if cast_missing else {}),
486
503
  },
487
504
  rationale=reason,
@@ -8,6 +8,7 @@ from .casting import DataTypeInference
8
8
  from .datetime_features import DateTimeExpander
9
9
  from .duplicates import DuplicateRowHandler, duplicate_report
10
10
  from .encoding import (
11
+ BinaryEncoder,
11
12
  CategoricalEncoder,
12
13
  FrequencyEncoder,
13
14
  OneHotEncoder,
@@ -51,6 +52,7 @@ __all__ = [
51
52
  "detect_outliers",
52
53
  "CategoricalEncoder",
53
54
  "OneHotEncoder",
55
+ "BinaryEncoder",
54
56
  "OrdinalEncoder",
55
57
  "FrequencyEncoder",
56
58
  "TargetEncoder",
@@ -33,6 +33,7 @@ from ..types import CATEGORICAL_LIKE, ModelFamily, SemanticType, Severity, Stage
33
33
  __all__ = [
34
34
  "RareCategoryGrouper",
35
35
  "OneHotEncoder",
36
+ "BinaryEncoder",
36
37
  "OrdinalEncoder",
37
38
  "FrequencyEncoder",
38
39
  "TargetEncoder",
@@ -336,6 +337,99 @@ class OneHotEncoder(_CategoricalBase):
336
337
  return out
337
338
 
338
339
 
340
+ class BinaryEncoder(_CategoricalBase):
341
+ """Expand each category into ``ceil(log2(n))`` binary indicator columns.
342
+
343
+ A middle ground between one-hot (``n`` columns) and ordinal (one column with a
344
+ false ordering). Unseen categories at transform time are encoded as all-zero,
345
+ consistent with :class:`OneHotEncoder`.
346
+ """
347
+
348
+ stage = Stage.ENCODE
349
+
350
+ def __init__(
351
+ self,
352
+ columns: Optional[Sequence[str]] = None,
353
+ dtype: str = "int8",
354
+ ) -> None:
355
+ super().__init__(columns)
356
+ self.dtype = dtype
357
+
358
+ def _fit(self, X: pd.DataFrame, y: Optional[pd.Series], context: FitContext) -> None:
359
+ self.categories_: Dict[str, List[Any]] = {}
360
+ self.mappings_: Dict[str, Dict[Any, int]] = {}
361
+ self.n_bits_: Dict[str, int] = {}
362
+ self.output_names_: Dict[str, List[str]] = {}
363
+
364
+ with context.journal.timer(self.stage, type(self).__name__, "fit", "fit") as timer:
365
+ total = 0
366
+ for column in self.columns_:
367
+ series = self._as_object(X[column])
368
+ categories = sorted(series.dropna().unique(), key=_sort_key)
369
+ self.categories_[column] = categories
370
+ self.mappings_[column] = {c: i for i, c in enumerate(categories)}
371
+ n_bits = int(np.ceil(np.log2(max(len(categories), 1))))
372
+ self.n_bits_[column] = n_bits
373
+ self.output_names_[column] = [f"{column}__bin{i}" for i in range(n_bits)]
374
+ total += n_bits
375
+
376
+ timer.columns = list(self.columns_)
377
+ timer.effect = {"n_output_columns": total}
378
+
379
+ def _transform(self, X: pd.DataFrame, context: FitContext) -> pd.DataFrame:
380
+ added: Dict[str, pd.Series] = {}
381
+ unknown_counts: Dict[str, int] = {}
382
+
383
+ with context.journal.timer(
384
+ self.stage, type(self).__name__, "binary", "transform"
385
+ ) as timer:
386
+ for column in self.columns_:
387
+ if column not in X.columns:
388
+ continue
389
+ series = self._as_object(X[column])
390
+ mapping = self.mappings_[column]
391
+ integer_codes = series.map(mapping)
392
+ unknown = series.notna() & integer_codes.isna()
393
+ n_unknown = int(unknown.sum())
394
+ if n_unknown:
395
+ unknown_counts[column] = n_unknown
396
+
397
+ known = integer_codes.notna()
398
+ codes_int = integer_codes[known].to_numpy(dtype=np.int64)
399
+ for bit, name in enumerate(self.output_names_[column]):
400
+ values = np.zeros(len(X), dtype=self.dtype)
401
+ if known.any():
402
+ values[known.to_numpy()] = ((codes_int >> bit) & 1).astype(self.dtype)
403
+ added[name] = pd.Series(values, index=X.index, name=name)
404
+
405
+ if unknown_counts:
406
+ context.journal.warn(
407
+ "unseen_categories",
408
+ f"{sum(unknown_counts.values())} value(s) across "
409
+ f"{len(unknown_counts)} column(s) were not present at fit time and "
410
+ f"were encoded as all-zero indicators: "
411
+ f"{', '.join(sorted(unknown_counts))}.",
412
+ Severity.INFO,
413
+ tuple(unknown_counts),
414
+ unknown_counts,
415
+ )
416
+
417
+ timer.columns = list(self.categories_)
418
+ timer.effect = {
419
+ "n_output_columns": len(added),
420
+ "n_unknown_values": unknown_counts,
421
+ }
422
+
423
+ remaining = {str(c): X[c] for c in X.columns if str(c) not in self.categories_}
424
+ return pd.DataFrame({**remaining, **added}, index=X.index, copy=False)
425
+
426
+ def _compute_feature_names_out(self) -> List[str]:
427
+ out = [c for c in self.feature_names_in_ if c not in self.categories_]
428
+ for column in self.categories_:
429
+ out.extend(self.output_names_[column])
430
+ return out
431
+
432
+
339
433
  class OrdinalEncoder(_CategoricalBase):
340
434
  """Map categories to integer codes.
341
435
 
@@ -734,13 +828,11 @@ class CategoricalEncoder(_CategoricalBase):
734
828
  if strategy == "target":
735
829
  return TargetEncoder(cols)
736
830
  if strategy == "binary":
737
- raise ConfigurationError(
738
- "encoding='binary' is not implemented in this version. Use 'frequency' "
739
- "or 'target' for high-cardinality columns; both produce a single "
740
- "column and are better understood."
741
- )
831
+ return BinaryEncoder(cols)
742
832
  raise ConfigurationError.unknown_option(
743
- "encoding", strategy, ["onehot", "ordinal", "frequency", "count", "target"]
833
+ "encoding",
834
+ strategy,
835
+ ["onehot", "ordinal", "frequency", "count", "target", "binary"],
744
836
  )
745
837
 
746
838
  def _fit_transform(
@@ -774,12 +866,17 @@ class CategoricalEncoder(_CategoricalBase):
774
866
  return X[keep]
775
867
 
776
868
  def _compute_feature_names_out(self) -> List[str]:
777
- # Must match _transform exactly. One-hot appends its indicator columns at the
778
- # end rather than expanding in place (expanding in place would mean rebuilding
869
+ # Must match _transform exactly. One-hot and binary append indicator columns at
870
+ # the end rather than expanding in place (expanding in place would mean rebuilding
779
871
  # the frame around each encoded column), so the names have to be appended too.
780
872
  # Ordinal, frequency and target encoding all replace their column in position.
781
873
  onehot = self.encoders_.get("onehot")
782
- encoded_away = set(getattr(onehot, "categories_", {})) if onehot else set()
874
+ binary = self.encoders_.get("binary")
875
+ encoded_away = set()
876
+ if onehot is not None:
877
+ encoded_away.update(onehot.categories_) # type: ignore[attr-defined]
878
+ if binary is not None:
879
+ encoded_away.update(binary.categories_) # type: ignore[attr-defined]
783
880
  names = [
784
881
  c
785
882
  for c in self.feature_names_in_
@@ -788,4 +885,7 @@ class CategoricalEncoder(_CategoricalBase):
788
885
  if onehot is not None:
789
886
  for column in onehot.categories_: # type: ignore[attr-defined]
790
887
  names.extend(onehot.output_names_[column]) # type: ignore[attr-defined]
888
+ if binary is not None:
889
+ for column in binary.categories_: # type: ignore[attr-defined]
890
+ names.extend(binary.output_names_[column]) # type: ignore[attr-defined]
791
891
  return names
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: edaprep
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Transparent, leakage-safe EDA and ML preprocessing with an explainable planner.
5
5
  Author-email: bijay <bijaybeezoe@gmail.com>
6
6
  License: MIT
@@ -321,7 +321,7 @@ changed what shipped are listed here alongside the commits.
321
321
  | [@qiaobochi040726-source](https://github.com/qiaobochi040726-source) | Removed `Config.n_jobs`, a setting the library accepted and never read ([#11](https://github.com/bijay-odyssey/edaprep/pull/11)) |
322
322
  | [@zbs-ops](https://github.com/zbs-ops) | Benchmarked parallelising the per-column profiling loop, which is what decided [#9](https://github.com/bijay-odyssey/edaprep/issues/9) — no commit, and the reason the parameter went |
323
323
  | [@Jeferson681](https://github.com/Jeferson681) | Made the missing-indicator and high-missing rules see gaps introduced at the cast step ([#16](https://github.com/bijay-odyssey/edaprep/pull/16)) |
324
- | [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one; it survives as [#18](https://github.com/bijay-odyssey/edaprep/issues/18) ([#17](https://github.com/bijay-odyssey/edaprep/pull/17)) |
324
+ | [@luziyi123448-gif](https://github.com/luziyi123448-gif) | Independent fix for the same issue whose design was the better one; that design shipped in [#18](https://github.com/bijay-odyssey/edaprep/issues/18) ([#17](https://github.com/bijay-odyssey/edaprep/pull/17)) |
325
325
  | [@LeonxLJX](https://github.com/LeonxLJX) | Formatted the tree and made `ruff format` a real CI gate rather than an advisory one ([#25](https://github.com/bijay-odyssey/edaprep/pull/25)) |
326
326
  | [@TrueFurina](https://github.com/TrueFurina) | First to run the Codecov upload path, which is how we learned it fails silently from a fork with no token — CI green, nothing reported; coverage moved to the job summary instead ([#21](https://github.com/bijay-odyssey/edaprep/pull/21)) |
327
327
 
@@ -794,6 +794,75 @@ def test_column_with_real_nan_still_gets_a_missing_indicator() -> None:
794
794
  assert int(out["amount__was_missing"].sum()) == 24
795
795
 
796
796
 
797
+ def test_missing_fraction_is_consistent_across_the_three_cast_missing_rules() -> None:
798
+ """#18: drop_high_missing, missing_indicator and impute_by_type must agree.
799
+
800
+ Regression: all three rules read cast-introduced placeholders (see #16 / #14), but
801
+ only stored the pre-cast ``cp.missing_fraction`` in ``params`` while their rationale
802
+ quoted the post-cast figure -- a decision whose prose says "above the 5.0% flag
803
+ threshold" reporting ``missing_fraction=0.0`` in the machine-readable half. One frame
804
+ with three columns, one per rule, and every decision must report the same
805
+ (post-cast) fraction plus the ``n_rows`` needed to reconstruct it.
806
+ """
807
+ gen = np.random.default_rng(41)
808
+ n = 300
809
+
810
+ def blanks(count: int) -> list:
811
+ values = [f"{v:.2f}" for v in gen.uniform(20, 8000, size=n)]
812
+ for i in range(count):
813
+ values[i] = ""
814
+ return values
815
+
816
+ frame = pd.DataFrame(
817
+ {
818
+ "col_impute": blanks(6), # 2%: below the 5% indicator threshold
819
+ "col_indicator": blanks(24), # 8%: indicator, then impute
820
+ "col_drop": blanks(210), # 70%: above the 60% drop ceiling
821
+ "y": gen.integers(0, 2, size=n),
822
+ }
823
+ )
824
+
825
+ pipe = AutoPipeline(target="y", model_family="linear", random_state=0)
826
+ pipe.fit(frame)
827
+ plan = pipe.plan_
828
+
829
+ expected = {
830
+ "col_impute": (6, 6 / n),
831
+ "col_indicator": (24, 24 / n),
832
+ "col_drop": (210, 210 / n),
833
+ }
834
+ tracked_actions = {"drop", "add_missing_indicator"}
835
+ for column, (cast_missing, fraction) in expected.items():
836
+ for decision in plan.decisions:
837
+ if decision.column != column:
838
+ continue
839
+ if decision.action not in tracked_actions and not decision.action.startswith(
840
+ "impute_"
841
+ ):
842
+ continue
843
+ assert decision.params.get("cast_missing") == cast_missing, decision
844
+ assert decision.params["missing_fraction"] == pytest.approx(fraction), decision
845
+ assert decision.params["n_rows"] == n, decision
846
+
847
+ actions = {d.column: d.action for d in plan.decisions if d.action.startswith("impute_")}
848
+ assert actions.get("col_impute", "").startswith("impute_")
849
+ assert actions.get("col_indicator", "").startswith("impute_")
850
+ assert "col_drop" not in actions, (
851
+ "a dropped column must not also be scheduled for imputation"
852
+ )
853
+ indicator_columns = {
854
+ d.column for d in plan.decisions if d.action == "add_missing_indicator"
855
+ }
856
+ assert indicator_columns == {"col_indicator"}
857
+ assert "col_drop" in plan.dropped_columns
858
+
859
+ # The new key must survive a JSON round-trip like every other param.
860
+ restored = Plan.from_dict(json.loads(plan.to_json()))
861
+ before = {(d.column, d.action): d.params for d in plan.decisions}
862
+ after = {(d.column, d.action): d.params for d in restored.decisions}
863
+ assert before == after
864
+
865
+
797
866
  def test_from_dict_tolerates_settings_this_version_removed() -> None:
798
867
  """A Config saved before a setting was retired must still load.
799
868
 
@@ -15,6 +15,7 @@ from edaprep.exceptions import (
15
15
  TransformationError,
16
16
  )
17
17
  from edaprep.preprocessing import (
18
+ BinaryEncoder,
18
19
  CategoricalEncoder,
19
20
  ColumnDropper,
20
21
  ConstantFilter,
@@ -396,6 +397,47 @@ def test_onehot_drop_first() -> None:
396
397
  assert list(out.columns) == ["c_b", "c_c"]
397
398
 
398
399
 
400
+ def test_binary_encoder_distinct_categories_get_distinct_codes() -> None:
401
+ frame = pd.DataFrame({"c": ["a", "b", "c", "d"]})
402
+ out = BinaryEncoder(["c"]).fit_transform(frame, None, ctx(frame))
403
+ codes = [tuple(row) for row in out.to_numpy()]
404
+ assert len(codes) == len(set(codes))
405
+
406
+
407
+ def test_binary_encoder_output_width() -> None:
408
+ for n in (1, 2, 3, 4, 5, 8, 9):
409
+ categories = [f"v{i}" for i in range(n)]
410
+ frame = pd.DataFrame({"c": categories})
411
+ out = BinaryEncoder(["c"]).fit_transform(frame, None, ctx(frame))
412
+ expected = int(np.ceil(np.log2(max(n, 1))))
413
+ assert len([c for c in out.columns if c.startswith("c__bin")]) == expected
414
+
415
+
416
+ def test_binary_column_order_is_deterministic() -> None:
417
+ gen = np.random.default_rng(81)
418
+ frame = pd.DataFrame({"c": gen.choice(list("dcba"), 100)})
419
+ a = BinaryEncoder(["c"]).fit_transform(frame, None, ctx(frame))
420
+ b = BinaryEncoder(["c"]).fit_transform(frame.iloc[::-1], None, ctx(frame))
421
+ assert list(a.columns) == list(b.columns)
422
+ assert list(a.columns) == ["c__bin0", "c__bin1"]
423
+
424
+
425
+ def test_binary_encoder_unseen_category_is_all_zero() -> None:
426
+ train = pd.DataFrame({"c": ["a", "b", "c", "d"] * 20})
427
+ context = ctx(train)
428
+ encoder = BinaryEncoder(["c"]).fit(train, None, context)
429
+ out = encoder.transform(pd.DataFrame({"c": ["zzz"]}), context)
430
+ assert out.to_numpy().tolist() == [[0, 0]]
431
+
432
+
433
+ def test_binary_encoder_feature_names_match_columns() -> None:
434
+ frame = pd.DataFrame({"c": ["a", "b", "c", "d", "e"]})
435
+ context = ctx(frame)
436
+ encoder = BinaryEncoder(["c"]).fit(frame, None, context)
437
+ out = encoder.transform(frame, context)
438
+ assert list(encoder.get_feature_names_out()) == list(out.columns)
439
+
440
+
399
441
  def test_ordinal_encoder_honours_an_ordered_categorical() -> None:
400
442
  dtype = pd.CategoricalDtype(["low", "medium", "high"], ordered=True)
401
443
  frame = pd.DataFrame({"c": pd.Series(["high", "low", "medium"], dtype=dtype)})
@@ -798,6 +840,7 @@ ALL_TRANSFORMERS = [
798
840
  OutlierHandler,
799
841
  CategoricalEncoder,
800
842
  OneHotEncoder,
843
+ BinaryEncoder,
801
844
  OrdinalEncoder,
802
845
  FrequencyEncoder,
803
846
  RareCategoryGrouper,
@@ -1 +0,0 @@
1
- __version__ = "0.2.1"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes