edaprep 0.2.2__tar.gz → 0.3.0__tar.gz

{edaprep-0.2.2/src/edaprep.egg-info → edaprep-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: edaprep
-Version: 0.2.2
+Version: 0.3.0
 Summary: Transparent, leakage-safe EDA and ML preprocessing with an explainable planner.
 Author-email: bijay <bijaybeezoe@gmail.com>
 License: MIT

edaprep-0.3.0/src/edaprep/_version.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{edaprep-0.2.2 → edaprep-0.3.0}/src/edaprep/config.py

@@ -36,7 +36,19 @@ __all__ = ["Config", "ColumnConfig", "Thresholds", "AUTO"]
 AUTO = "auto"
 
 _MISSING_STRATEGIES = frozenset(
-    {AUTO, "mean", "median", "mode", "constant", "ffill", "bfill", "drop_rows", "none"}
+    {
+        AUTO,
+        "mean",
+        "median",
+        "mode",
+        "constant",
+        "ffill",
+        "bfill",
+        "drop_rows",
+        "none",
+        "knn",
+        "iterative",
+    }
 )
 _OUTLIER_METHODS = frozenset({AUTO, "iqr", "zscore", "modified_zscore", "percentile", "none"})
 _OUTLIER_STRATEGIES = frozenset(
@@ -383,7 +395,12 @@ class Config:
         """Return the override record for ``name`` without creating one."""
         return self.columns.get(name)
 
-    def set_columns(self, overrides: Mapping[str, Mapping[str, Any]]) -> "Config":
+    def set_columns(
+        self,
+        overrides: Mapping[str, Mapping[str, Any]],
+        *,
+        warn_on_unknown: bool = False,
+    ) -> "Config":
         """Bulk-apply overrides: ``{"age": {"imputation": "median"}, ...}``."""
         for name, kwargs in overrides.items():
             col = self.column(name)
@@ -392,6 +409,15 @@ class Config:
                     valid = [
                         f.name for f in dataclasses.fields(ColumnConfig) if f.name != "name"
                     ]
+                    if warn_on_unknown:
+                        warnings.warn(
+                            f"Config.from_dict ignoring unrecognised column({name!r}) "
+                            f"setting: {key!r}. It was either retired in a later "
+                            f"version of edaprep or is misspelt.",
+                            UserWarning,
+                            stacklevel=2,
+                        )
+                        continue
                     raise ConfigurationError.unknown_option(
                         f"column({name!r}) setting", key, valid
                     )
@@ -484,7 +510,21 @@ class Config:
         equally likely to be a typo, and this is the only signal you would get.
         """
         data = dict(data)
-        thresholds = Thresholds(**data.pop("thresholds", {}))
+        thresholds_raw = dict(data.pop("thresholds", {}))
+        known_thresholds = {f.name for f in dataclasses.fields(Thresholds)}
+        unknown_thresholds = sorted(set(thresholds_raw) - known_thresholds)
+        if unknown_thresholds:
+            warnings.warn(
+                f"Config.from_dict ignoring {len(unknown_thresholds)} unrecognised "
+                f"threshold setting(s): "
+                f"{', '.join(repr(k) for k in unknown_thresholds)}. They were either "
+                f"retired in a later version of edaprep or are misspelt; the rest of "
+                f"the threshold configuration was applied unchanged.",
+                UserWarning,
+                stacklevel=2,
+            )
+            thresholds_raw = {k: v for k, v in thresholds_raw.items() if k in known_thresholds}
+        thresholds = Thresholds(**thresholds_raw)
         columns_raw = data.pop("columns", {})
 
         known = {f.name for f in dataclasses.fields(cls)}
@@ -504,7 +544,7 @@ class Config:
         for name, kwargs in columns_raw.items():
             kwargs = dict(kwargs)
             kwargs.pop("name", None)
-            cfg.set_columns({name: kwargs})
+            cfg.set_columns({name: kwargs}, warn_on_unknown=True)
         return cfg
 
     def __repr__(self) -> str:

{edaprep-0.2.2 → edaprep-0.3.0}/src/edaprep/eda/categorical.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 from typing import List, Optional
 
+import numpy as np
 import pandas as pd
 
 from ..config import Config
@@ -26,7 +27,7 @@ def categorical_summary(
     rare_threshold = config.effective_rare_threshold
     high_cardinality = config.effective_high_cardinality
     n_rows = profile.n_rows
-    floor = max(1, int(rare_threshold * n_rows))
+    floor = max(1, int(np.ceil(rare_threshold * n_rows)))
 
     rows: List[dict] = []
     for name in profile.column_order:

{edaprep-0.2.2 → edaprep-0.3.0}/src/edaprep/planning/rules.py

@@ -452,6 +452,11 @@ def _rule_impute(cp: ColumnProfile, ctx: RuleContext) -> Optional[Decision]:
             f"{cast_missing} placeholder value(s) become NaN when the column is cast, "
             f"so it needs imputation despite reporting 0.0% missing"
         )
+    elif outlier_may_impute:
+        found = (
+            "0.0% missing today, but outlier_strategy='impute' may introduce NaN at the "
+            "OUTLIERS stage that runs before this one"
+        )
     else:
         found = f"{_pct(cp.missing_fraction)} missing"
 

{edaprep-0.2.2 → edaprep-0.3.0}/src/edaprep/preprocessing/casting.py

@@ -146,7 +146,7 @@ class DataTypeInference(Transformer, ColumnTransformerMixin):
                     stripped = series.astype(object).map(
                         lambda v: v.strip() if isinstance(v, str) else v
                     )
-                    counts["stripped"] = int((stripped != series).sum())
+                    counts["stripped"] = int(((stripped != series) & series.notna()).sum())
                     series = stripped
 
                 if "sentinels_to_nan" in actions:

{edaprep-0.2.2 → edaprep-0.3.0}/src/edaprep/preprocessing/datetime_features.py

@@ -43,13 +43,13 @@ _EXTRACTORS: Dict[str, Callable[[pd.Series], pd.Series]] = {
     "hour": lambda s: s.dt.hour,
     "minute": lambda s: s.dt.minute,
     "second": lambda s: s.dt.second,
-    "is_weekend": lambda s: (s.dt.dayofweek >= 5).astype("float64"),
-    "is_month_start": lambda s: s.dt.is_month_start.astype("float64"),
-    "is_month_end": lambda s: s.dt.is_month_end.astype("float64"),
-    "is_quarter_start": lambda s: s.dt.is_quarter_start.astype("float64"),
-    "is_quarter_end": lambda s: s.dt.is_quarter_end.astype("float64"),
-    "is_year_start": lambda s: s.dt.is_year_start.astype("float64"),
-    "is_year_end": lambda s: s.dt.is_year_end.astype("float64"),
+    "is_weekend": lambda s: (s.dt.dayofweek >= 5).astype("float64").where(s.notna()),
+    "is_month_start": lambda s: s.dt.is_month_start.astype("float64").where(s.notna()),
+    "is_month_end": lambda s: s.dt.is_month_end.astype("float64").where(s.notna()),
+    "is_quarter_start": lambda s: s.dt.is_quarter_start.astype("float64").where(s.notna()),
+    "is_quarter_end": lambda s: s.dt.is_quarter_end.astype("float64").where(s.notna()),
+    "is_year_start": lambda s: s.dt.is_year_start.astype("float64").where(s.notna()),
+    "is_year_end": lambda s: s.dt.is_year_end.astype("float64").where(s.notna()),
     "month_sin": lambda s: np.sin(2 * np.pi * s.dt.month / 12.0),
     "month_cos": lambda s: np.cos(2 * np.pi * s.dt.month / 12.0),
     "dayofweek_sin": lambda s: np.sin(2 * np.pi * s.dt.dayofweek / 7.0),

{edaprep-0.2.2 → edaprep-0.3.0}/src/edaprep/preprocessing/encoding.py

@@ -437,6 +437,12 @@ class OrdinalEncoder(_CategoricalBase):
     subset of codes, and for genuinely ordered columns.  It imposes a false ordering on
     a nominal column fed to a linear or distance-based model, which is why the planner
     only selects it for ``model_family="tree"`` or ``SemanticType.ORDINAL``.
+
+    The ``dtype`` parameter (default ``"int32"``) is applied to encoded columns. When
+    missing values are present and ``dtype`` cannot represent NA (such as standard numpy
+    integer types), the column safely falls back to ``"float64"`` so downstream imputers
+    can process it. Nullable integer dtypes (e.g. ``"Int32"``) are preserved with missing
+    values.
     """
 
     stage = Stage.ENCODE
@@ -497,7 +503,13 @@ class OrdinalEncoder(_CategoricalBase):
                 # NaN in the input stays NaN so a later imputer can see it; only
                 # genuinely unseen categories get the sentinel code.
                 codes = codes.mask(unknown, self.unknown_value)
-                replacements[column] = codes.astype("float64")
+                if codes.isna().any():
+                    try:
+                        replacements[column] = codes.astype(self.dtype)
+                    except (ValueError, TypeError):
+                        replacements[column] = codes.astype("float64")
+                else:
+                    replacements[column] = codes.astype(self.dtype)
 
             if unknown_counts:
                 context.journal.warn(

{edaprep-0.2.2 → edaprep-0.3.0}/src/edaprep/preprocessing/missing.py

@@ -14,7 +14,7 @@ column that is mostly missing is reported rather than quietly invented.
 
 from __future__ import annotations
 
-from typing import Any, Dict, List, Optional, Sequence
+from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Type
 
 import numpy as np
 import pandas as pd
@@ -31,6 +31,45 @@ __all__ = ["MissingValueHandler", "MissingIndicator"]
 _LEARNED = frozenset({"mean", "median", "mode"})
 #: Strategies applied row-wise at transform time with no learned state.
 _ROWWISE = frozenset({"ffill", "bfill"})
+#: Multivariate imputers (scikit-learn, optional dependency).
+_SKLEARN_IMPUTE = frozenset({"knn", "iterative"})
+
+_ADVANCED_INSTALL_MSG = (
+    "Install the optional dependency with: pip install 'edaprep[advanced]' "
+    "(requires scikit-learn>=1.1)."
+)
+
+
+def _valid_imputation_strategies() -> List[str]:
+    return sorted(
+        _LEARNED
+        | _ROWWISE
+        | _SKLEARN_IMPUTE
+        | {"constant", "missing_category", "none", "drop_rows"}
+    )
+
+
+def _is_numeric_imputation_column(series: pd.Series) -> bool:
+    return pd.api.types.is_numeric_dtype(series.dtype) and not pd.api.types.is_bool_dtype(
+        series.dtype
+    )
+
+
+def _load_sklearn_imputers() -> Tuple[Type[Any], Type[Any]]:
+    try:
+        from sklearn.impute import KNNImputer
+    except ImportError as exc:  # pragma: no cover - exercised via mock in tests
+        raise ConfigurationError(
+            f"strategy='knn' or 'iterative' requires scikit-learn. {_ADVANCED_INSTALL_MSG}"
+        ) from exc
+    try:
+        from sklearn.experimental import enable_iterative_imputer  # noqa: F401
+        from sklearn.impute import IterativeImputer
+    except ImportError as exc:  # pragma: no cover
+        raise ConfigurationError(
+            f"strategy='iterative' requires scikit-learn. {_ADVANCED_INSTALL_MSG}"
+        ) from exc
+    return KNNImputer, IterativeImputer
 
 
 class MissingValueHandler(Transformer, ColumnTransformerMixin):
@@ -43,7 +82,8 @@ class MissingValueHandler(Transformer, ColumnTransformerMixin):
         (robust to the skew real tabular data is full of), mode for categorical
         and binary, and an explicit ``"missing"`` category for high-cardinality
         categoricals where the mode is not representative.  Any other value pins
-        every column to that strategy.
+        every column to that strategy.  ``"knn"`` and ``"iterative"`` use
+        scikit-learn multivariate imputers (``edaprep[advanced]``).
     fill_value :
         Used by ``strategy="constant"``.
     per_column :
@@ -117,6 +157,118 @@ class MissingValueHandler(Transformer, ColumnTransformerMixin):
             return "mode"
         return "missing_category"
 
+    def _eligible_numeric_columns(self, X: pd.DataFrame, context: FitContext) -> List[str]:
+        """All numeric columns in ``X`` usable as multivariate predictors (ordered)."""
+        target = context.target
+        eligible: List[str] = []
+        for column in map(str, X.columns):
+            if column == target:
+                continue
+            if not _is_numeric_imputation_column(X[column]):
+                continue
+            eligible.append(column)
+        return eligible
+
+    def _require_numeric_for_sklearn(self, column: str, series: pd.Series) -> None:
+        if _is_numeric_imputation_column(series):
+            return
+        raise ConfigurationError(
+            f"strategy='knn' and 'iterative' are only valid for numeric columns; "
+            f"column {column!r} has dtype {series.dtype}. Use 'mode', "
+            f"'missing_category', or 'median' as appropriate."
+        )
+
+    def _block_matrix(self, X: pd.DataFrame, block: Sequence[str]) -> np.ndarray:
+        """Build a float matrix aligned with ``block`` for sklearn imputers."""
+        n_rows = len(X)
+        matrix = np.empty((n_rows, len(block)), dtype=np.float64)
+        for j, column in enumerate(block):
+            values = pd.to_numeric(X[column], errors="coerce").to_numpy(dtype=np.float64)
+            matrix[:, j] = values
+        return matrix
+
+    def _fit_sklearn_imputers(self, X: pd.DataFrame, context: FitContext) -> None:
+        needs_knn = any(s == "knn" for s in self.strategies_.values())
+        needs_iterative = any(s == "iterative" for s in self.strategies_.values())
+        self.knn_impute_columns_: List[str] = [
+            c for c in self.columns_ if self.strategies_.get(c) == "knn"
+        ]
+        self.iterative_impute_columns_: List[str] = [
+            c for c in self.columns_ if self.strategies_.get(c) == "iterative"
+        ]
+
+        if not needs_knn and not needs_iterative:
+            self.imputer_block_columns_: List[str] = []
+            self.imputer_block_all_missing_: Set[str] = set()
+            self.knn_imputer_ = None
+            self.iterative_imputer_ = None
+            return
+
+        KNNImputer, IterativeImputer = _load_sklearn_imputers()
+        eligible = self._eligible_numeric_columns(X, context)
+        self.imputer_block_all_missing_ = {
+            column for column in eligible if len(X) and int(X[column].isna().sum()) == len(X)
+        }
+        block = [column for column in eligible if column not in self.imputer_block_all_missing_]
+        self.imputer_block_columns_ = block
+
+        if not block:
+            self.knn_imputer_ = None
+            self.iterative_imputer_ = None
+            return
+
+        fit_matrix = self._block_matrix(X, block)
+
+        if needs_knn:
+            self.knn_imputer_ = KNNImputer()
+            self.knn_imputer_.fit(fit_matrix)
+        else:
+            self.knn_imputer_ = None
+
+        if needs_iterative:
+            self.iterative_imputer_ = IterativeImputer(
+                random_state=context.config.random_state,
+                sample_posterior=False,
+            )
+            self.iterative_imputer_.fit(fit_matrix)
+        else:
+            self.iterative_imputer_ = None
+
+    def _apply_sklearn_imputations(
+        self,
+        X: pd.DataFrame,
+        replacements: Dict[str, pd.Series],
+        filled_counts: Dict[str, int],
+    ) -> None:
+        block = self.imputer_block_columns_
+        if not block:
+            return
+
+        transform_matrix = self._block_matrix(X, block)
+        block_index = {name: idx for idx, name in enumerate(block)}
+
+        for imputer, output_columns in (
+            (self.knn_imputer_, self.knn_impute_columns_),
+            (self.iterative_imputer_, self.iterative_impute_columns_),
+        ):
+            if imputer is None:
+                continue
+            imputed = imputer.transform(transform_matrix)
+            for column in output_columns:
+                if column not in X.columns or column not in block_index:
+                    continue
+                series = X[column]
+                mask = series.isna()
+                n_missing = int(mask.sum())
+                if n_missing == 0:
+                    continue
+                col_idx = block_index[column]
+                filled = series.copy()
+                filled.loc[mask] = imputed[mask.to_numpy(), col_idx]
+                replacements[column] = filled
+                after = int(filled.isna().sum())
+                filled_counts[column] = n_missing - after
+
     def _fit(self, X: pd.DataFrame, y: Optional[pd.Series], context: FitContext) -> None:
         self.strategies_: Dict[str, str] = {}
         self.fill_values_: Dict[str, Any] = {}
@@ -137,6 +289,19 @@ class MissingValueHandler(Transformer, ColumnTransformerMixin):
                 if strategy in _ROWWISE:
                     continue
 
+                if strategy in _SKLEARN_IMPUTE:
+                    self._require_numeric_for_sklearn(column, series)
+                    if n_missing == len(X):
+                        context.journal.warn(
+                            "no_data_to_learn_fill",
+                            f"Column {column!r} is entirely missing in the training data, "
+                            f"so multivariate imputation cannot learn a fill for it. "
+                            f"Missing values in this column will be left as-is.",
+                            Severity.WARNING,
+                            (column,),
+                        )
+                    continue
+
                 if strategy == "constant":
                     if self.fill_value is None:
                         override = context.config.get_column(column)
@@ -170,6 +335,8 @@ class MissingValueHandler(Transformer, ColumnTransformerMixin):
                         {"missing_fraction": round(missing_fraction, 4)},
                     )
 
+            self._fit_sklearn_imputers(X, context)
+
             timer.columns = list(self.columns_)
             timer.params = {"strategy": self.strategy}
             timer.effect = {
@@ -214,7 +381,7 @@ class MissingValueHandler(Transformer, ColumnTransformerMixin):
         raise ConfigurationError.unknown_option(
             "imputation strategy",
             strategy,
-            sorted(_LEARNED | _ROWWISE | {"constant", "missing_category", "none"}),
+            _valid_imputation_strategies(),
         )
 
     def _transform(self, X: pd.DataFrame, context: FitContext) -> pd.DataFrame:
@@ -225,6 +392,8 @@ class MissingValueHandler(Transformer, ColumnTransformerMixin):
         with context.journal.timer(
             self.stage, type(self).__name__, "impute", "transform"
         ) as timer:
+            self._apply_sklearn_imputations(X, replacements, filled_counts)
+
             for column in self.columns_:
                 if column not in X.columns:
                     continue
@@ -239,6 +408,9 @@ class MissingValueHandler(Transformer, ColumnTransformerMixin):
                 if self.add_indicator and n_missing:
                     added[f"{column}__was_missing"] = mask.astype(np.int8)
 
+                if strategy in _SKLEARN_IMPUTE:
+                    continue
+
                 if n_missing == 0:
                     continue
 
@@ -252,12 +424,13 @@ class MissingValueHandler(Transformer, ColumnTransformerMixin):
                         continue
                     replacements[column] = _fill(series, value)
 
-                after = (
-                    int(replacements[column].isna().sum())
-                    if column in replacements
-                    else n_missing
-                )
-                filled_counts[column] = n_missing - after
+                if column not in filled_counts:
+                    after = (
+                        int(replacements[column].isna().sum())
+                        if column in replacements
+                        else n_missing
+                    )
+                    filled_counts[column] = n_missing - after
 
             timer.columns = sorted(filled_counts)
             timer.effect = {

{edaprep-0.2.2 → edaprep-0.3.0}/src/edaprep/preprocessing/text.py

@@ -106,9 +106,15 @@ class TextColumnHandler(Transformer, ColumnTransformerMixin):
 
         added: Dict[str, pd.Series] = {}
         for column in present:
-            as_str = X[column].astype(str)
-            added[f"{column}__length"] = as_str.str.len().astype("float64")
-            added[f"{column}__n_words"] = as_str.str.split().str.len().astype("float64")
+            series = X[column]
+            mask = series.isna()
+            as_str = series.dropna().astype(str)
+            length = as_str.str.len().astype("float64").reindex(series.index)
+            n_words = as_str.str.split().str.len().astype("float64").reindex(series.index)
+            length[mask] = float("nan")
+            n_words[mask] = float("nan")
+            added[f"{column}__length"] = length
+            added[f"{column}__n_words"] = n_words
         keep = {str(c): X[c] for c in X.columns if str(c) not in present}
         return pd.DataFrame({**keep, **added}, index=X.index, copy=False)
 

{edaprep-0.2.2 → edaprep-0.3.0/src/edaprep.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: edaprep
-Version: 0.2.2
+Version: 0.3.0
 Summary: Transparent, leakage-safe EDA and ML preprocessing with an explainable planner.
 Author-email: bijay <bijaybeezoe@gmail.com>
 License: MIT

{edaprep-0.2.2 → edaprep-0.3.0}/tests/test_eda.py

@@ -418,3 +418,29 @@ def test_config_repr_is_readable() -> None:
     text = repr(config)
     assert "random_state=1" in text
     assert "column override" in text
+
+
+def test_categorical_summary_rare_floor_matches_planner() -> None:
+    """categorical_summary and the planner must agree on the rare-level floor when
+    threshold * n_rows is fractional."""
+    from edaprep.eda.categorical import categorical_summary
+    from edaprep.planning.planner import Planner
+    from edaprep.profiling.profiler import profile
+
+    # threshold=0.01 (default), n_rows=999 -> 0.01 * 999 = 9.99, floor should be ceil(9.99) = 10
+    # 5 levels so planner does not skip due to low cardinality (< 5)
+    data = ["cat_a"] * 250 + ["cat_b"] * 250 + ["cat_c"] * 250 + ["cat_d"] * 240 + ["rare"] * 9
+    df = pd.DataFrame({"c": data, "y": [0] * 999})
+    config = Config(model_family="linear")
+    prof = profile(df, target="y", config=config)
+
+    cat_table = categorical_summary(df, prof, config)
+    row = cat_table.set_index("column").loc["c"]
+    assert row["n_rare_levels"] == 1
+
+    plan = Planner(config).plan(prof)
+    rare_step = [
+        d for d in plan.decisions if d.action == "group_rare_categories" and d.column == "c"
+    ]
+    assert len(rare_step) == 1
+    assert rare_step[0].params["min_count"] == 10

{edaprep-0.2.2 → edaprep-0.3.0}/tests/test_leakage.py

@@ -129,6 +129,44 @@ def test_scaler_uses_train_statistics_only(split_frame) -> None:
     np.testing.assert_allclose(out["num"].to_numpy(), expected.to_numpy())
 
 
+def test_imputer_knn_transform_is_independent_of_batching(split_frame) -> None:
+    train, test = split_frame
+    train = train.copy()
+    test = test.copy()
+    train.loc[train.index[:40], "num"] = np.nan
+    test.loc[test.index[:15], "num"] = np.nan
+
+    context = _context(train)
+    handler = MissingValueHandler(["num", "skewed"], strategy="knn").fit(
+        train, train["y"], context
+    )
+    whole = handler.transform(test, context)
+    row_by_row = pd.concat(
+        [handler.transform(test.iloc[[i]], context) for i in range(len(test))],
+        axis=0,
+    )
+    pd.testing.assert_frame_equal(whole, row_by_row)
+
+
+def test_imputer_iterative_transform_is_independent_of_batching(split_frame) -> None:
+    train, test = split_frame
+    train = train.copy()
+    test = test.copy()
+    train.loc[train.index[:40], "num"] = np.nan
+    test.loc[test.index[:15], "num"] = np.nan
+
+    context = _context(train)
+    handler = MissingValueHandler(["num", "skewed"], strategy="iterative").fit(
+        train, train["y"], context
+    )
+    whole = handler.transform(test, context)
+    row_by_row = pd.concat(
+        [handler.transform(test.iloc[[i]], context) for i in range(len(test))],
+        axis=0,
+    )
+    pd.testing.assert_frame_equal(whole, row_by_row)
+
+
 def test_imputer_uses_train_median_only(split_frame) -> None:
     train, test = split_frame
     train = train.copy()

{edaprep-0.2.2 → edaprep-0.3.0}/tests/test_pipeline.py

@@ -678,6 +678,24 @@ def test_column_with_no_missing_and_no_placeholders_plans_no_imputation() -> Non
     assert not any(a.startswith("impute_") for a in actions), actions
 
 
+def test_impute_rationale_names_outlier_strategy_when_nothing_is_missing() -> None:
+    """Regression for #47: imputation planned only because outliers may introduce NaN
+    must say so, rather than reporting a bare (and correct, but uninformative) 0%."""
+    gen = np.random.default_rng(5)
+    frame = pd.DataFrame(
+        {"amount": gen.normal(50, 10, size=100), "y": gen.integers(0, 2, size=100)}
+    )
+    assert frame["amount"].isna().sum() == 0
+
+    plan = Planner(Config(random_state=0, outlier_strategy="impute")).plan(
+        profile(frame, target="y")
+    )
+    decision = next(
+        d for d in plan.decisions if d.column == "amount" and d.action.startswith("impute_")
+    )
+    assert "outlier" in decision.rationale
+
+
 def test_placeholder_strings_get_a_missing_indicator() -> None:
     """Placeholder strings become NaN at Stage.CAST, so MISSING_FLAG must plan a flag.
 
@@ -897,3 +915,37 @@ def test_from_dict_round_trip_is_warning_free() -> None:
     assert restored.random_state == 7
     assert restored.verbose is True
     assert restored.column("age").imputation == "median"
+
+
+def test_from_dict_tolerates_retired_thresholds() -> None:
+    """Config.from_dict must drop unrecognised Thresholds settings with a warning."""
+    data = Config().to_dict()
+    data["thresholds"]["old_removed_field"] = 0.5
+
+    with pytest.warns(UserWarning, match="old_removed_field"):
+        restored = Config.from_dict(data)
+
+    assert isinstance(restored, Config)
+    assert not hasattr(restored.thresholds, "old_removed_field")
+
+
+def test_from_dict_tolerates_retired_column_settings() -> None:
+    """Config.from_dict must drop unrecognised per-column settings with a warning."""
+    data = Config().to_dict()
+    data["columns"] = {"age": {"imputation": "median", "old_removed_col_field": "x"}}
+
+    with pytest.warns(UserWarning, match="old_removed_col_field"):
+        restored = Config.from_dict(data)
+
+    assert isinstance(restored, Config)
+    col = restored.get_column("age")
+    assert col is not None
+    assert col.imputation == "median"
+    assert not hasattr(col, "old_removed_col_field")
+
+
+def test_set_columns_still_raises_on_unknown_setting_by_default() -> None:
+    """Direct user-facing set_columns still raises on typo."""
+    config = Config()
+    with pytest.raises(ConfigurationError):
+        config.set_columns({"age": {"typo_option": "bad"}})

{edaprep-0.2.2 → edaprep-0.3.0}/tests/test_transformers.py

@@ -362,6 +362,191 @@ def test_missing_indicator_respects_threshold() -> None:
     assert indicator.columns_ == []
 
 
+def _correlated_imputation_frame() -> pd.DataFrame:
+    rng = np.random.default_rng(21)
+    n = 120
+    y = rng.normal(0, 1, n)
+    x = 2.0 * y + rng.normal(0, 0.05, n)
+    frame = pd.DataFrame({"x": x, "y": y})
+    frame.loc[frame.index[::4], "x"] = np.nan
+    return frame
+
+
+def test_knn_imputation_uses_correlated_feature_not_median() -> None:
+    frame = _correlated_imputation_frame()
+    context = ctx(frame)
+    median_out = MissingValueHandler(["x"], strategy="median").fit_transform(
+        frame, None, context
+    )
+    knn_out = MissingValueHandler(["x"], strategy="knn").fit_transform(frame, None, context)
+    mask = frame["x"].isna()
+    median_fill = float(frame["x"].median())
+    imputed = knn_out.loc[mask, "x"].to_numpy()
+    expected = (2.0 * frame.loc[mask, "y"]).to_numpy()
+    assert np.corrcoef(imputed, expected)[0, 1] > 0.95
+    assert not np.allclose(imputed, median_fill, rtol=1e-6, atol=1e-6)
+    assert not np.allclose(imputed, median_out.loc[mask, "x"].to_numpy(), rtol=1e-6, atol=1e-6)
+
+
+def test_iterative_imputation_uses_correlated_feature_not_median() -> None:
+    frame = _correlated_imputation_frame()
+    context = ctx(frame)
+    median_out = MissingValueHandler(["x"], strategy="median").fit_transform(
+        frame, None, context
+    )
+    iterative_out = MissingValueHandler(["x"], strategy="iterative").fit_transform(
+        frame, None, context
+    )
+    mask = frame["x"].isna()
+    median_fill = float(frame["x"].median())
+    imputed = iterative_out.loc[mask, "x"].to_numpy()
+    expected = (2.0 * frame.loc[mask, "y"]).to_numpy()
+    assert np.corrcoef(imputed, expected)[0, 1] > 0.95
+    assert not np.allclose(imputed, median_fill, rtol=1e-6, atol=1e-6)
+    assert not np.allclose(imputed, median_out.loc[mask, "x"].to_numpy(), rtol=1e-6, atol=1e-6)
+
+
+def test_knn_uses_numeric_predictor_outside_columns_and_leaves_it_unchanged() -> None:
+    """Only ``x`` is handled, but ``y`` in ``X`` must still inform KNN imputation."""
+    frame = _correlated_imputation_frame()
+    context = ctx(frame)
+    handler = MissingValueHandler(["x"], strategy="knn").fit(frame, None, context)
+    assert "y" not in handler.columns_
+    assert handler.imputer_block_columns_ == ["x", "y"]
+    out = handler.transform(frame, context)
+    mask = frame["x"].isna()
+    imputed = out.loc[mask, "x"].to_numpy()
+    expected = (2.0 * frame.loc[mask, "y"]).to_numpy()
+    assert np.corrcoef(imputed, expected)[0, 1] > 0.95
+    assert np.allclose(out["y"].to_numpy(), frame["y"].to_numpy())
+    assert list(out.columns) == list(frame.columns)
+
+
+def test_knn_median_predictor_column_stays_in_imputer_block() -> None:
+    """Column y uses median but must still be a KNN predictor for x."""
+    frame = _correlated_imputation_frame()
+    context = ctx(frame)
+    handler = MissingValueHandler(["x", "y"], strategy="median", per_column={"x": "knn"}).fit(
+        frame, None, context
+    )
+    assert handler.imputer_block_columns_ == ["x", "y"]
+    assert handler.strategies_["y"] == "median"
+    out = handler.transform(frame, context)
+    mask = frame["x"].isna()
+    imputed = out.loc[mask, "x"].to_numpy()
+    expected = (2.0 * frame.loc[mask, "y"]).to_numpy()
+    assert np.corrcoef(imputed, expected)[0, 1] > 0.95
+    assert np.allclose(out["y"].to_numpy(), frame["y"].to_numpy())
+
+
+def test_all_missing_predictor_column_is_excluded_and_does_not_distort_knn() -> None:
+    frame = _correlated_imputation_frame()
+    with_dead = frame.assign(z=np.nan)
+    context = ctx(with_dead)
+    with_z = MissingValueHandler(["x"], strategy="knn").fit_transform(with_dead, None, context)
+    without_z = MissingValueHandler(["x"], strategy="knn").fit_transform(frame, None, context)
+    handler = MissingValueHandler(["x"], strategy="knn").fit(with_dead, None, context)
+    assert "z" in handler.imputer_block_all_missing_
+    assert "z" not in handler.imputer_block_columns_
+    mask = frame["x"].isna()
+    np.testing.assert_allclose(with_z.loc[mask, "x"], without_z.loc[mask, "x"], rtol=1e-10)
+    assert with_z["z"].isna().all()
+
+
+def test_all_missing_knn_target_column_stays_missing_with_warning() -> None:
+    frame = pd.DataFrame({"x": [np.nan] * 12, "y": np.arange(12.0)})
+    context = ctx(frame)
+    handler = MissingValueHandler(["x"], strategy="knn").fit(frame, None, context)
+    assert "x" in handler.imputer_block_all_missing_
+    assert "x" not in handler.imputer_block_columns_
+    assert any(w.code == "no_data_to_learn_fill" for w in context.journal.warnings)
+    out = handler.transform(frame, context)
+    assert out["x"].isna().all()
+    assert list(out.columns) == list(frame.columns)
+    assert np.allclose(out["y"].to_numpy(), frame["y"].to_numpy())
+
+
+def test_knn_transform_batch_equals_row_by_row() -> None:
+    frame = _correlated_imputation_frame()
+    context = ctx(frame)
+    handler = MissingValueHandler(["x", "y"], strategy="knn").fit(frame, None, context)
+    whole = handler.transform(frame, context)
+    rows = pd.concat(
+        [handler.transform(frame.iloc[[i]], context) for i in range(len(frame))],
+        axis=0,
+    )
+    pd.testing.assert_frame_equal(whole, rows)
+
+
+def test_iterative_transform_batch_equals_row_by_row() -> None:
+    frame = _correlated_imputation_frame()
+    context = ctx(frame)
+    handler = MissingValueHandler(["x", "y"], strategy="iterative").fit(frame, None, context)
+    whole = handler.transform(frame, context)
+    rows = pd.concat(
+        [handler.transform(frame.iloc[[i]], context) for i in range(len(frame))],
+        axis=0,
+    )
+    pd.testing.assert_frame_equal(whole, rows)
+
+
+def test_sklearn_imputers_are_not_refit_on_transform() -> None:
+    frame = _correlated_imputation_frame()
+    context = ctx(frame)
+    handler = MissingValueHandler(["x", "y"], strategy="knn").fit(frame, None, context)
+    assert handler.knn_imputer_ is not None
+
+    def _no_knn_fit(*args, **kwargs):
+        raise AssertionError("KNNImputer.fit must not run during transform")
+
+    handler.knn_imputer_.fit = _no_knn_fit  # type: ignore[method-assign]
+    handler.transform(frame.iloc[:10], context)
+
+    handler2 = MissingValueHandler(["x", "y"], strategy="iterative").fit(frame, None, context)
+    assert handler2.iterative_imputer_ is not None
+
+    def _no_iter_fit(*args, **kwargs):
+        raise AssertionError("IterativeImputer.fit must not run during transform")
+
+    handler2.iterative_imputer_.fit = _no_iter_fit  # type: ignore[method-assign]
+    handler2.transform(frame.iloc[:10], context)
+
+
+def test_knn_imputation_rejects_non_numeric_column() -> None:
+    frame = pd.DataFrame({"c": ["a", None, "b"], "x": [1.0, np.nan, 3.0]})
+    with pytest.raises(ConfigurationError, match="numeric"):
+        MissingValueHandler(["c"], strategy="knn").fit(frame, None, ctx(frame))
+
+
+def test_iterative_imputation_rejects_non_numeric_column() -> None:
+    frame = pd.DataFrame({"c": ["a", None, "b"], "x": [1.0, np.nan, 3.0]})
+    with pytest.raises(ConfigurationError, match="numeric"):
+        MissingValueHandler(["c"], strategy="iterative").fit(frame, None, ctx(frame))
+
+
+def test_sklearn_imputation_requires_advanced_extra(monkeypatch) -> None:
+    import edaprep.preprocessing.missing as missing_mod
+
+    def _missing_sklearn():
+        raise ConfigurationError(
+            "strategy='knn' or 'iterative' requires scikit-learn. "
+            "Install the optional dependency with: pip install 'edaprep[advanced]' "
+            "(requires scikit-learn>=1.1)."
+        )
+
+    monkeypatch.setattr(missing_mod, "_load_sklearn_imputers", _missing_sklearn)
+    frame = pd.DataFrame({"x": [1.0, np.nan, 3.0]})
+    with pytest.raises(ConfigurationError, match="edaprep\\[advanced\\]"):
+        MissingValueHandler(["x"], strategy="knn").fit(frame, None, ctx(frame))
+
+
+def test_config_accepts_knn_and_iterative_missing_strategies() -> None:
+    Config(missing_strategy="knn")
+    Config(missing_strategy="iterative")
+    Config().column("age").imputation = "knn"
+    Config().column("score").imputation = "iterative"
+
+
 # ============================== encoding ==============================================
 
 
@@ -455,6 +640,39 @@ def test_ordinal_encoder_marks_unseen_but_keeps_nan_as_nan() -> None:
     assert pd.isna(out["c"].tolist()[2])  # missing stays missing
 
 
+def test_ordinal_encoder_dtype_applied_when_no_missing() -> None:
+    frame = pd.DataFrame({"c": ["apple", "banana", "cherry"]})
+    context = ctx(frame)
+    out_default = OrdinalEncoder(["c"]).fit_transform(frame, None, context)
+    assert out_default["c"].dtype == "int32"
+    assert out_default["c"].tolist() == [0, 1, 2]
+
+    out_int16 = OrdinalEncoder(["c"], dtype="int16").fit_transform(frame, None, context)
+    assert out_int16["c"].dtype == "int16"
+
+
+def test_ordinal_encoder_dtype_fallback_on_missing_and_nullable_support() -> None:
+    train = pd.DataFrame({"c": ["apple", "banana", "cherry"] * 5})
+    context = ctx(train)
+    encoder_int32 = OrdinalEncoder(["c"], dtype="int32").fit(train, None, context)
+
+    # Column with NaN: numpy int32 cannot represent NaN -> safely falls back to float64
+    eval_frame = pd.DataFrame({"c": ["apple", "unknown_fruit", None]})
+    out_fallback = encoder_int32.transform(eval_frame, context)
+    assert out_fallback["c"].dtype == "float64"
+    assert out_fallback["c"].iloc[0] == 0.0
+    assert out_fallback["c"].iloc[1] == -1.0  # unseen
+    assert pd.isna(out_fallback["c"].iloc[2])
+
+    # Nullable Int32: supports missing values -> retains Int32 dtype
+    encoder_nullable = OrdinalEncoder(["c"], dtype="Int32").fit(train, None, context)
+    out_nullable = encoder_nullable.transform(eval_frame, context)
+    assert str(out_nullable["c"].dtype) == "Int32"
+    assert out_nullable["c"].iloc[0] == 0
+    assert out_nullable["c"].iloc[1] == -1
+    assert pd.isna(out_nullable["c"].iloc[2])
+
+
 def test_frequency_encoder_and_unseen_categories() -> None:
     train = pd.DataFrame({"c": ["a"] * 70 + ["b"] * 30})
     context = ctx(train)
@@ -611,6 +829,43 @@ def test_datetime_rejects_unknown_features() -> None:
         DateTimeExpander(["d"], features=["nonsense"]).fit(frame, None, ctx(frame))
 
 
+def test_datetime_boolean_features_retain_nan_for_missing_dates() -> None:
+    boolean_features = [
+        "is_weekend",
+        "is_month_start",
+        "is_month_end",
+        "is_quarter_start",
+        "is_quarter_end",
+        "is_year_start",
+        "is_year_end",
+    ]
+    frame = pd.DataFrame(
+        {"d": pd.to_datetime(["2024-01-01", None, "2024-03-01", "2024-01-06"])}
+    )
+    context = ctx(frame)
+    expander = DateTimeExpander(
+        ["d"],
+        features=["dayofweek", *boolean_features],
+        drop_constant=False,
+    ).fit(frame, None, context)
+    out = expander.transform(frame, context)
+
+    # Row 1 is NaT: all boolean features must be NaN, not 0.0 or False
+    for feat in boolean_features:
+        col = f"d__{feat}"
+        assert col in out.columns
+        assert np.isnan(out.loc[1, col]), f"{col} should be NaN for NaT row"
+
+    # Row 0 is 2024-01-01 (Monday, month start, quarter start, year start):
+    assert out.loc[0, "d__is_weekend"] == 0.0
+    assert out.loc[0, "d__is_month_start"] == 1.0
+    assert out.loc[0, "d__is_quarter_start"] == 1.0
+    assert out.loc[0, "d__is_year_start"] == 1.0
+
+    # Row 3 is 2024-01-06 (Saturday):
+    assert out.loc[3, "d__is_weekend"] == 1.0
+
+
 # ============================== casting ===============================================
 
 
@@ -627,6 +882,16 @@ def test_whitespace_is_stripped() -> None:
     assert out["c"].tolist() == ["USA", "usa", "USA"]
 
 
+def test_whitespace_stripped_count_ignores_missing_values() -> None:
+    frame = pd.DataFrame({"c": [" A ", "B", None, " C "]})
+    context = ctx(frame)
+
+    DataTypeInference(["c"]).fit_transform(frame, None, context)
+
+    entry = context.journal.transform_entries()[-1]
+    assert entry.effect["per_column"]["c"]["stripped"] == 2
+
+
 def test_numeric_strings_are_parsed() -> None:
     gen = np.random.default_rng(15)
     frame = pd.DataFrame({"n": [f"{v:.2f}" for v in gen.normal(50, 10, 200)]})
@@ -830,6 +1095,21 @@ def test_text_length_features() -> None:
     assert "t__length" in out.columns and "t__n_words" in out.columns
 
 
+def test_text_length_features_missing() -> None:
+    frame = pd.DataFrame({"t": ["hello world", "a longer sentence here", None, "short", None]})
+    out = TextColumnHandler(strategy="length_features", columns=["t"]).fit_transform(
+        frame, None, ctx(frame)
+    )
+    assert out["t__length"].iloc[0] == 11.0
+    assert out["t__n_words"].iloc[0] == 2.0
+    assert pd.isna(out["t__length"].iloc[2])
+    assert pd.isna(out["t__n_words"].iloc[2])
+    assert pd.isna(out["t__length"].iloc[4])
+    assert pd.isna(out["t__n_words"].iloc[4])
+    assert out["t__length"].iloc[3] == 5.0
+    assert out["t__n_words"].iloc[3] == 1.0
+
+
 # ============================== the contract ===========================================
 
 

edaprep-0.2.2/src/edaprep/_version.py

@@ -1 +0,0 @@
-__version__ = "0.2.2"