SuperModelingFactory 0.5.6__tar.gz → 0.5.8__tar.gz

{supermodelingfactory-0.5.6 → supermodelingfactory-0.5.8}/Modeling_Tool/Core/Parallel_ODPS_Manager.py

@@ -64,6 +64,26 @@ def _strip_trailing_semicolon(sql: str) -> str:
     return sql.strip().rstrip(";").strip()
 
 
+def _first_error_line(exc: BaseException, limit: int = 200) -> str:
+    """Return ``ExcType: first non-empty message line`` for embedding in a
+    re-raised error message, truncated to ``limit`` chars.
+
+    ODPS errors carry the actionable ``ODPS-XXXXXXX: ...`` description on
+    their first line followed by long instance/traceback detail; only the
+    first line is worth surfacing inline.
+    """
+    for line in str(exc).splitlines():
+        line = line.strip()
+        if line:
+            break
+    else:
+        line = ""
+    text = f"{type(exc).__name__}: {line}" if line else type(exc).__name__
+    if len(text) > limit:
+        return text[:limit] + "..."
+    return text
+
+
 def _pull_one_chunk(
     chunk_spec: dict[str, Any],
     runner: ODPSRunner | None,
@@ -218,8 +238,9 @@ class ParallelODPSManager:
             self.odps_runner.run_sql(probe_sql, to_df=True, n_process=1)
         except Exception as exc:
             raise ValueError(
-                f"unique_key validation failed for pull SQL. "
-                f"Check that unique_key={self.config.unique_key!r} is visible in the SQL scope."
+                f"unique_key validation failed for pull SQL: {_first_error_line(exc)}. "
+                f"Common cause: unique_key={self.config.unique_key!r} not visible in "
+                f"the SQL scope; see the chained exception for the full ODPS error."
             ) from exc
 
     def _build_hash_pull_chunks(

{supermodelingfactory-0.5.6 → supermodelingfactory-0.5.8}/Modeling_Tool/Feature/Feature_Screen.py

@@ -413,7 +413,17 @@ def _weighted_woe_bins_screen(
             if var not in ins.columns or ins[var].nunique(dropna=False) <= 1:
                 continue
             bins_ins = _bins_to_numpy(adapter.assign_bins(ins, var))
-            x_ins = ins[var].to_numpy(dtype=float)
+            x_series = ins[var]
+            if pd.api.types.is_numeric_dtype(x_series):
+                x_ins = x_series.to_numpy(dtype=float)
+            else:
+                # Categorical/object feature: IV itself is computed from the
+                # already-assigned bins; ``x`` only feeds np.isfinite() inside
+                # _weighted_iv_from_assigned_bins to derive the weighted
+                # missing rate. A hard float cast crashes on string levels
+                # (e.g. '4.高中'), so encode observed -> 1.0 / missing -> NaN,
+                # matching the notna semantics of _missing_rate_for_series.
+                x_ins = np.where(x_series.notna().to_numpy(), 1.0, np.nan)
             iv_val, n_b, miss = _weighted_iv_from_assigned_bins(y_ins, w_ins, bins_ins, x_ins)
             iv_records.append({
                 "var": var,

{supermodelingfactory-0.5.6 → supermodelingfactory-0.5.8}/Modeling_Tool/Feature/Weighted_Screen.py

@@ -383,6 +383,28 @@ def _weighted_corr_for_screen(
             if cols:
                 X = woe_ins[cols].to_numpy(dtype=float)
                 return _weighted_pearson_corr_matrix(X, w_ins, nan_policy="pairwise")
+    non_numeric = [v for v in current if not pd.api.types.is_numeric_dtype(ins[v])]
+    if non_numeric:
+        # Raw-value Pearson correlation is undefined for categorical/object
+        # features. Exclude them from the correlation computation but keep
+        # them in the returned matrix as NaN rows/columns: NaN never exceeds
+        # the dedup threshold, so categorical features survive the corr stage
+        # untouched instead of crashing the float cast.
+        warnings.warn(
+            f"raw-value correlation skips {len(non_numeric)} non-numeric "
+            f"feature(s) {non_numeric[:5]}; they are kept through the corr "
+            f"stage. Set corr_use_woe_bins=True to correlate categorical "
+            f"features via their WOE encoding.",
+            UserWarning,
+            stacklevel=2,
+        )
+        corr_full = np.full((len(current), len(current)), np.nan)
+        numeric_idx = [i for i, v in enumerate(current) if v not in set(non_numeric)]
+        if numeric_idx:
+            X_num = ins[[current[i] for i in numeric_idx]].astype(float).to_numpy()
+            corr_num = _weighted_pearson_corr_matrix(X_num, w_ins, nan_policy=corr_nan_policy)
+            corr_full[np.ix_(numeric_idx, numeric_idx)] = corr_num
+        return corr_full
     X = ins[current].astype(float).to_numpy()
     return _weighted_pearson_corr_matrix(X, w_ins, nan_policy=corr_nan_policy)
 

{supermodelingfactory-0.5.6 → supermodelingfactory-0.5.8}/Modeling_Tool/Pipeline/feature_validation.py

@@ -1231,6 +1231,10 @@ class FeatureValidationPipeline:
                 "missing_rate_ref",
                 cfg.woe_params.get("missing_ref_value", -999999),
             ),
+            # Without this, a selection run that has to fit its own screening
+            # binner (no prefit WOE engine) would bin declared categorical
+            # features as numeric and crash on string levels.
+            "categorical_features": params.get("categorical_features", cfg.categorical_features),
         }
         return screen_config_from_mapping(
             mapping,

{supermodelingfactory-0.5.6 → supermodelingfactory-0.5.8}/Modeling_Tool/Pipeline/field_meta.py

@@ -198,6 +198,7 @@ _FIELD_LABELS = {
     "train_ri_models": "训练 RI 后模型",
     "ri_model_type": "RI 后模型类型",
     "ri_model_params": "RI 后模型参数",
+    "lr_nan_handling": "lr 模型缺失值处理",
     "include_no_ri_benchmark": "加入无 RI benchmark",
     "ri_validation_frac": "RI 验证集比例",
     "write_ri_datasets": "写出 RI 增强样本",
@@ -352,6 +353,7 @@ _FIELD_OPTIONS = {
     "feature_selection_mode": ["run", "from_artifact", "skip"],
     "prescore_model_type": ["lgb", "xgb", "cat", "lr"],
     "ri_model_type": ["lgb", "xgb", "cat", "lr"],
+    "lr_nan_handling": ["fillna_median", "fillna_mean", "fillna_0", "raise"],
     "ri_methods": ["simple_augment", "hard_cutoff", "fuzzy_augment", "parceling"],
     "ri_score_direction": ["high_bad", "high_good"],
     "ri_approved_scope": ["reference_only", "output_subset"],

{supermodelingfactory-0.5.6 → supermodelingfactory-0.5.8}/Modeling_Tool/Pipeline/reject_inference.py

@@ -20,6 +20,48 @@ from ._common import (
 )
 
 
+class _LRNanFillWrapper:
+    """Wrap a fitted LRMaster with train-derived NaN/Inf fill values.
+
+    sklearn's LogisticRegression rejects non-finite inputs while the RI
+    pipeline feeds models raw feature frames, so the exact fill values
+    learned from the training frame must be re-applied at every predict
+    call — otherwise training and scoring would silently run on different
+    imputations. Module-level class so saved models stay picklable.
+    """
+
+    def __init__(self, model: Any, fill_values: dict[str, float], nan_handling: str):
+        self._inner = model
+        self.fill_values_ = dict(fill_values)
+        self.nan_handling = str(nan_handling)
+
+    def _fill(self, data: pd.DataFrame) -> pd.DataFrame:
+        fill_cols = [c for c in self.fill_values_ if c in data.columns]
+        if not fill_cols:
+            return data
+        filled = data.copy()
+        block = filled[fill_cols].replace([np.inf, -np.inf], np.nan)
+        filled[fill_cols] = block.fillna(self.fill_values_)
+        return filled
+
+    def predict_proba(self, data: pd.DataFrame, varlist: list[str] | None = None, **kwargs: Any) -> Any:
+        return self._inner.predict_proba(self._fill(data), varlist=varlist, **kwargs)
+
+    def predict(self, data: pd.DataFrame, varlist: list[str] | None = None, **kwargs: Any) -> Any:
+        return self._inner.predict(self._fill(data), varlist=varlist, **kwargs)
+
+    def __getattr__(self, name: str) -> Any:
+        # Never delegate dunders: leaking the inner model's __getstate__ /
+        # __setstate__ into pickle would serialize the wrapper as a bare
+        # LRMaster and drop the fill values on reload.
+        if name.startswith("__") and name.endswith("__"):
+            raise AttributeError(name)
+        inner = self.__dict__.get("_inner")
+        if inner is None:
+            raise AttributeError(name)
+        return getattr(inner, name)
+
+
 @dataclass
 class RejectInferencePipelineConfig:
     output_dir: str = "output/reject_inference"
@@ -46,6 +88,12 @@ class RejectInferencePipelineConfig:
     train_ri_models: bool = True
     ri_model_type: str = "lgb"
     ri_model_params: dict[str, Any] = field(default_factory=dict)
+    # NaN/Inf handling for lr models (both prescore_model_type="lr" and
+    # ri_model_type="lr"). GBM backends accept NaN natively; sklearn
+    # LogisticRegression does not, so lr features are imputed with
+    # train-derived fill values that are re-applied at scoring time.
+    # "raise" forbids imputation and fails fast on non-finite features.
+    lr_nan_handling: Literal["fillna_median", "fillna_mean", "fillna_0", "raise"] = "fillna_median"
     include_no_ri_benchmark: bool = True
     ri_validation_frac: float = 0.2
     save_models: bool = False
@@ -239,6 +287,9 @@ class RejectInferencePipeline:
         if model_type == "lr":
             lr_params = dict(params)
             standardize = bool(lr_params.pop("standardize", False))
+            train_fit, val_fit, fill_values = self._prepare_lr_frames(
+                train_fit, val_fit, feature_cols,
+            )
             model = LRMaster(params=lr_params or None, standardize=standardize)
             model.fit(
                 data=train_fit,
@@ -249,6 +300,8 @@ class RejectInferencePipeline:
                 val_tgt_name=cfg.target_col,
                 weight_col=weight_col,
             )
+            if fill_values is not None:
+                return _LRNanFillWrapper(model, fill_values, cfg.lr_nan_handling)
             return model
 
         model = GradientBoostingModel(model_type, params)
@@ -264,6 +317,69 @@ class RejectInferencePipeline:
         )
         return model
 
+    def _prepare_lr_frames(
+        self,
+        train_fit: pd.DataFrame,
+        val_fit: pd.DataFrame,
+        feature_cols: list[str],
+    ) -> tuple[pd.DataFrame, pd.DataFrame, dict[str, float] | None]:
+        """Apply cfg.lr_nan_handling to the lr feature frames.
+
+        GBM backends consume NaN natively; sklearn LogisticRegression raises
+        ``Input X contains NaN`` on raw pipeline features. Fill values are
+        computed on the training frame only and returned so the fitted model
+        re-applies the identical imputation at scoring time via
+        _LRNanFillWrapper. Callers pass throwaway copies, so frames are
+        modified in place.
+        """
+        cfg = self.config
+        allowed = {"fillna_median", "fillna_mean", "fillna_0", "raise"}
+        mode = str(cfg.lr_nan_handling)
+        if mode not in allowed:
+            raise ValueError(
+                f"lr_nan_handling must be one of {sorted(allowed)}; got {mode!r}"
+            )
+        train_block = train_fit[feature_cols].replace([np.inf, -np.inf], np.nan)
+        val_block = val_fit[feature_cols].replace([np.inf, -np.inf], np.nan)
+        n_bad_train = int(train_block.isna().sum().sum())
+        n_bad_val = int(val_block.isna().sum().sum())
+        if mode == "raise":
+            if n_bad_train or n_bad_val:
+                bad_cols = sorted(
+                    set(train_block.columns[train_block.isna().any()])
+                    | set(val_block.columns[val_block.isna().any()])
+                )
+                raise ValueError(
+                    f"lr model features contain {n_bad_train + n_bad_val} "
+                    f"non-finite value(s) across {len(bad_cols)} feature(s) "
+                    f"(e.g. {bad_cols[:5]}) and lr_nan_handling='raise'. "
+                    f"Clean the features first, or set lr_nan_handling to "
+                    f"'fillna_median'/'fillna_mean'/'fillna_0'."
+                )
+            return train_fit, val_fit, None
+        if mode == "fillna_0":
+            fill_series = pd.Series(0.0, index=feature_cols)
+        elif mode == "fillna_mean":
+            fill_series = train_block.mean(numeric_only=True).reindex(feature_cols).fillna(0.0)
+        else:
+            fill_series = train_block.median(numeric_only=True).reindex(feature_cols).fillna(0.0)
+        fill_values = {str(col): float(fill_series[col]) for col in feature_cols}
+        if n_bad_train or n_bad_val:
+            n_cols = int((train_block.isna().any() | val_block.isna().any()).sum())
+            warnings.warn(
+                f"lr model: {n_bad_train + n_bad_val} non-finite feature "
+                f"value(s) across {n_cols}/{len(feature_cols)} feature(s) "
+                f"filled with train-derived {mode} values (sklearn LR cannot "
+                f"handle NaN). Fill values are stored on the model and "
+                f"re-applied at scoring time; set lr_nan_handling='raise' to "
+                f"forbid imputation.",
+                UserWarning,
+                stacklevel=2,
+            )
+        train_fit[feature_cols] = train_block.fillna(fill_values)
+        val_fit[feature_cols] = val_block.fillna(fill_values)
+        return train_fit, val_fit, fill_values
+
     def run(self, data: pd.DataFrame) -> RejectInferencePipelineResult:
         cfg = self.config
         feature_cols = self._resolve_feature_cols(data)
@@ -1074,6 +1190,10 @@ class RejectInferencePipeline:
             "model_type": model_type,
             "random_state": cfg.random_state,
         }
+        fill_values = getattr(model, "fill_values_", None)
+        if fill_values is not None:
+            metadata["lr_nan_handling"] = getattr(model, "nan_handling", None)
+            metadata["lr_fill_values"] = dict(fill_values)
         save_model(
             model,
             path,

{supermodelingfactory-0.5.6 → supermodelingfactory-0.5.8}/Modeling_Tool/__init__.py

@@ -1,7 +1,7 @@
 # encoding: utf-8
 
 __author__ = "Jingkai Sun"
-__version__ = "0.5.6"
+__version__ = "0.5.8"
 
 from ._utils import SMF_MISSING_BIN
 

{supermodelingfactory-0.5.6 → supermodelingfactory-0.5.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: SuperModelingFactory
-Version: 0.5.6
+Version: 0.5.8
 Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
 Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
 Author: Kyle Sun
@@ -265,7 +265,7 @@ em.close_workbook()
 
 ## 版本
 
-- **Version**: 0.5.6
+- **Version**: 0.5.8
 - **Author**: Jingkai Sun
 
 ## 许可证

{supermodelingfactory-0.5.6 → supermodelingfactory-0.5.8}/README.md

@@ -208,7 +208,7 @@ em.close_workbook()
 
 ## 版本
 
-- **Version**: 0.5.6
+- **Version**: 0.5.8
 - **Author**: Jingkai Sun
 
 ## 许可证

{supermodelingfactory-0.5.6 → supermodelingfactory-0.5.8}/SuperModelingFactory.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: SuperModelingFactory
-Version: 0.5.6
+Version: 0.5.8
 Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
 Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
 Author: Kyle Sun
@@ -265,7 +265,7 @@ em.close_workbook()
 
 ## 版本
 
-- **Version**: 0.5.6
+- **Version**: 0.5.8
 - **Author**: Jingkai Sun
 
 ## 许可证

{supermodelingfactory-0.5.6 → supermodelingfactory-0.5.8}/pyproject.toml

@@ -12,7 +12,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "SuperModelingFactory"
-version = "0.5.6"
+version = "0.5.8"
 description = "Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting."
 readme = "README.md"
 requires-python = ">=3.10"

{supermodelingfactory-0.5.6 → supermodelingfactory-0.5.8}/setup.py

@@ -15,7 +15,7 @@ def _read(path: str) -> str:
 
 setup(
     name="SuperModelingFactory",
-    version=os.environ.get("SMF_VERSION", "0.5.6"),
+    version=os.environ.get("SMF_VERSION", "0.5.8"),
     description="Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.",
     long_description=_read("README.md"),
     long_description_content_type="text/markdown",