SuperModelingFactory 0.3.14__tar.gz → 0.4.2__tar.gz

{supermodelingfactory-0.3.14 → supermodelingfactory-0.4.2}/Modeling_Tool/Core/ODPS_Tool.py

@@ -247,7 +247,7 @@ class ODPSRunner(object):
 
         return table_schema
 
-    def upload_df(self, df, table_name, table_schema=None, partition=None):
+    def upload_df(self, df, table_name, table_schema=None, partition=None, atomic=True):
         """上传数据集至mc中创建新表
 
         Parameters
@@ -260,20 +260,113 @@ class ODPSRunner(object):
             数据集
         partition: string
             保存分区
+        atomic: bool, default True
+            When ``True`` (recommended, default from 0.4.2), the target table is
+            replaced through a temp-table + rename swap so that a failure between
+            ``delete_table`` and the completed write never leaves the caller
+            with a dropped-and-empty table. When ``False``, the pre-0.4.2
+            behaviour is used: the existing table is dropped first and then
+            recreated, so any exception during write leaves the target table
+            missing. Only set ``atomic=False`` if you deliberately want the
+            legacy behaviour, e.g. for tables whose downstream readers expect
+            a specific object identity.
         """
         if table_schema is None:
             df.loc[:, "py_inserttime"] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
             table_schema = self.cre_table_schema(df=df, partition_name=None)
 
-        self.o.delete_table(table_name, if_exists=True)
-        t = self.o.create_table(table_name, table_schema)
+        if not atomic:
+            # Legacy pre-0.4.2 non-atomic path — retained under an explicit
+            # opt-out for callers that need the old identity semantics.
+            self.o.delete_table(table_name, if_exists=True)
+            t = self.o.create_table(table_name, table_schema)
+            if bool(partition):
+                with t.open_writer(partition=partition, create_partition=True) as writer:
+                    writer.write(df.values.tolist())
+            else:
+                with t.open_writer() as writer:
+                    writer.write(df.values.tolist())
+            logger.info(f'<<<< 完成数据入表{table_name}: shape={df.shape} >>>>')
+            return
+
+        # Atomic swap path (default from 0.4.2). If any step before the final
+        # swap fails, the original table is untouched.
+        ts = datetime.now().strftime('%Y%m%d%H%M%S%f')
+        tmp_name = f"{table_name}__tmp_{ts}"
+        old_name = f"{table_name}__old_{ts}"
+
+        # Step 1: write data into the temp table. If this fails, drop the
+        # partial temp table and re-raise; the target table is untouched.
+        self.o.delete_table(tmp_name, if_exists=True)
+        tmp_table = self.o.create_table(tmp_name, table_schema)
+        try:
+            if bool(partition):
+                with tmp_table.open_writer(partition=partition, create_partition=True) as writer:
+                    writer.write(df.values.tolist())
+            else:
+                with tmp_table.open_writer() as writer:
+                    writer.write(df.values.tolist())
+        except Exception:
+            try:
+                self.o.delete_table(tmp_name, if_exists=True)
+            except Exception as cleanup_exc:  # noqa: BLE001
+                logger.warning(
+                    f"upload_df: failed to drop temp table {tmp_name} after write error: {cleanup_exc!r}"
+                )
+            raise
+
+        # Step 2: swap. If the original table exists, rename it aside first so
+        # that the rename of tmp -> target is guaranteed to succeed. Then drop
+        # the old copy (best-effort, warn on failure).
+        target_exists = self.o.exist_table(table_name)
+        if target_exists:
+            try:
+                self.o.run_sql(f"ALTER TABLE {table_name} RENAME TO {old_name};")
+            except Exception:
+                # Rename of the live target failed — do not touch it. Drop the
+                # tmp table so we don't leak it, then re-raise.
+                try:
+                    self.o.delete_table(tmp_name, if_exists=True)
+                except Exception as cleanup_exc:  # noqa: BLE001
+                    logger.warning(
+                        f"upload_df: failed to drop temp table {tmp_name} after rename-target failure: {cleanup_exc!r}"
+                    )
+                raise
+
+        try:
+            self.o.run_sql(f"ALTER TABLE {tmp_name} RENAME TO {table_name};")
+        except Exception:
+            # tmp -> target rename failed. If we already moved the original,
+            # try to restore it so the caller is not left with a missing table.
+            if target_exists:
+                try:
+                    self.o.run_sql(f"ALTER TABLE {old_name} RENAME TO {table_name};")
+                    logger.warning(
+                        f"upload_df: swap of {tmp_name} -> {table_name} failed; restored original from {old_name}"
+                    )
+                except Exception as restore_exc:  # noqa: BLE001
+                    logger.error(
+                        f"upload_df: swap of {tmp_name} -> {table_name} failed AND restore of "
+                        f"{old_name} -> {table_name} failed: {restore_exc!r}. "
+                        f"Manual recovery required — original data is in {old_name}."
+                    )
+            try:
+                self.o.delete_table(tmp_name, if_exists=True)
+            except Exception:  # noqa: BLE001
+                pass
+            raise
+
+        # Step 3: drop the moved-aside original — best-effort, do not fail the
+        # call if this cleanup errors, since the swap already succeeded.
+        if target_exists:
+            try:
+                self.o.delete_table(old_name, if_exists=True)
+            except Exception as cleanup_exc:  # noqa: BLE001
+                logger.warning(
+                    f"upload_df: swap succeeded but failed to drop old copy {old_name}: {cleanup_exc!r}. "
+                    f"Safe to delete manually."
+                )
 
-        if bool(partition):
-            with t.open_writer(partition=partition, create_partition=True) as writer:
-                writer.write(df.values.tolist())
-        else:
-            with t.open_writer() as writer:
-                writer.write(df.values.tolist())
         logger.info(f'<<<< 完成数据入表{table_name}: shape={df.shape} >>>>')
 
     def insert_df(self, df, table_name, overwrite=True, partition=None):

{supermodelingfactory-0.3.14 → supermodelingfactory-0.4.2}/Modeling_Tool/Feature/Distribution_Tool.py

@@ -186,6 +186,40 @@ def proc_means_by_grp(data, varlist, groupby=None, spec_missing_value=None, q=No
     return means_rpt
 
 
+_SCREENING_MEANS_COLS = ["attribute", "N_ALL", "N", "MISSING_RATE", "MIN", "MEAN", "MAX"]
+
+
+def proc_means_for_screening(data, varlist, spec_missing_value=None, q=None):
+    """Build a uniform means table for IV screening reports.
+
+    ``proc_means_by_grp`` returns object stats (unique/top/freq) when numeric and
+    categorical columns are melted together.  This helper splits by dtype so MIN,
+    MEAN, and MAX are always present (NaN for non-numeric features).
+    """
+    if not varlist:
+        return pd.DataFrame(columns=_SCREENING_MEANS_COLS)
+
+    numeric = [v for v in varlist if v in data.columns and pd.api.types.is_numeric_dtype(data[v])]
+    other = [v for v in varlist if v not in numeric and v in data.columns]
+    frames = []
+
+    if numeric:
+        frames.append(proc_means_by_grp(data, numeric, spec_missing_value=spec_missing_value, q=q))
+
+    for var in other:
+        part = proc_means_by_grp(data, [var], spec_missing_value=spec_missing_value, q=q)
+        for col in ("MIN", "MEAN", "MAX"):
+            if col not in part.columns:
+                part[col] = np.nan
+        frames.append(part)
+
+    if not frames:
+        return pd.DataFrame(columns=_SCREENING_MEANS_COLS)
+
+    out = pd.concat(frames, ignore_index=True)
+    return out[_SCREENING_MEANS_COLS]
+
+
 class DistributionShiftAnalyzer:
     """分布偏移分析器。
     

{supermodelingfactory-0.3.14 → supermodelingfactory-0.4.2}/Modeling_Tool/Feature/Distribution_Tool.pyi

@@ -25,6 +25,7 @@ class proc_means:
     def group_sum(self): ...
     def __call__(self, q = None): ...
 def proc_means_by_grp(data, varlist, groupby = None, spec_missing_value = None, q = None): ...
+def proc_means_for_screening(data, varlist, spec_missing_value = None, q = None): ...
 
 class DistributionShiftAnalyzer:
     def __init__(self, data, grp_name, benchmark_value): ...

{supermodelingfactory-0.3.14 → supermodelingfactory-0.4.2}/Modeling_Tool/Feature/Feature_Insights.py

@@ -6,7 +6,7 @@
 import pandas as pd
 from tqdm import tqdm
 
-from .Distribution_Tool import proc_means_by_grp
+from .Distribution_Tool import proc_means_by_grp, proc_means_for_screening
 import logging
 logger = logging.getLogger(__name__)
 
@@ -208,14 +208,15 @@ class VarExtractionInsights:
         high_iv_summary = iv_info_res.query(f"IV >= {iv_cut}").round(4)
         high_iv_varlist = high_iv_summary['var'].tolist()
 
-        means = proc_means_by_grp(data, high_iv_varlist, spec_missing_value=self.missing_rate_ref)
-
         if len(high_iv_varlist) == 0:
             logger.info(f"WARNING: No variable with IV >= {iv_cut}")
-            means = means.rename(columns={"index": "attribute"})
+
+        means = proc_means_for_screening(
+            data, high_iv_varlist, spec_missing_value=self.missing_rate_ref,
+        )
 
         fnl_summary = high_iv_summary.merge(
-            means[['attribute', 'N_ALL', 'N', 'MISSING_RATE', 'MIN', 'MEAN', 'MAX']],
+            means,
             left_on=['var'],
             right_on=['attribute'],
             how='left'

{supermodelingfactory-0.3.14 → supermodelingfactory-0.4.2}/Modeling_Tool/Feature/Feature_Screen.py

@@ -13,6 +13,7 @@ from Modeling_Tool.Core.sample_weight_utils import resolve_sample_weight
 
 from .Weighted_Screen import (
     WeightedScreenResult,
+    _apply_missing_rate_stage,
     _apply_stage_keep,
     _corr_dedup_weighted,
     _legacy_unweighted_screen,
@@ -58,6 +59,8 @@ class FeatureScreenConfig:
     corr_use_woe_bins: bool = False
     corr_nan_policy: Literal["pairwise", "median_fill", "raise"] = "pairwise"
     on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn"
+    missing_rate_threshold: float | None = None
+    missing_rate_ref: float | int = -999999
     woe_engine: str = "equal_freq"
     woe_fit_query: str | None = None
     woe_params: dict[str, Any] = field(
@@ -105,6 +108,8 @@ def screen_config_from_mapping(
         corr_use_woe_bins=bool(cfg.get("corr_use_woe_bins", False)),
         corr_nan_policy=str(cfg.get("corr_nan_policy", "pairwise")),  # type: ignore[arg-type]
         on_empty_stage=str(cfg.get("on_empty_stage", "keep_all_warn")),  # type: ignore[arg-type]
+        missing_rate_threshold=cfg.get("missing_rate_threshold"),
+        missing_rate_ref=cfg.get("missing_rate_ref", -999999),
         woe_engine=str(woe_engine or cfg.get("woe_engine", "equal_freq")),
         woe_fit_query=woe_fit_query if woe_fit_query is not None else cfg.get("woe_fit_query"),
         woe_params=dict(FeatureScreenConfig().woe_params | dict(woe_params or cfg.get("woe_params") or {})),
@@ -211,6 +216,14 @@ def _woe_bins_unweighted_screen(
     ins, oos, oot = splits["ins"], splits["oos"], splits["oot"]
     current = list(feature_cols)
     summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, None)]
+    current, missing_rate_table, missing_rate_dropped = _apply_missing_rate_stage(
+        ins,
+        current,
+        summary_rows,
+        missing_rate_threshold=config.missing_rate_threshold,
+        missing_rate_ref=config.missing_rate_ref,
+        on_empty_stage=config.on_empty_stage,
+    )
     binner = _resolve_screening_binner(splits, feature_cols, target_col, config, prefit_woe_engine)
 
     psi_table = pd.DataFrame(columns=["var", "psi_ins_oos", "psi_ins_oot", "psi_max"])
@@ -303,6 +316,8 @@ def _woe_bins_unweighted_screen(
         psi_table=psi_table,
         corr_dropped=corr_dropped,
         summary=pd.DataFrame(summary_rows),
+        missing_rate_table=missing_rate_table,
+        missing_rate_dropped=missing_rate_dropped,
     )
 
 
@@ -328,6 +343,15 @@ def _weighted_woe_bins_screen(
     y_ins = ins[target_col].astype(float).to_numpy()
     current = list(feature_cols)
     summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, weight_col)]
+    current, missing_rate_table, missing_rate_dropped = _apply_missing_rate_stage(
+        ins,
+        current,
+        summary_rows,
+        missing_rate_threshold=config.missing_rate_threshold,
+        missing_rate_ref=config.missing_rate_ref,
+        weight_col=weight_col,
+        on_empty_stage=config.on_empty_stage,
+    )
 
     binner = prefit_woe_engine
     if binner is None:
@@ -438,6 +462,8 @@ def _weighted_woe_bins_screen(
         psi_table=psi_table,
         corr_dropped=corr_dropped,
         summary=pd.DataFrame(summary_rows),
+        missing_rate_table=missing_rate_table,
+        missing_rate_dropped=missing_rate_dropped,
     )
 
 
@@ -485,6 +511,8 @@ def feature_screen(
             corr_nan_policy=cfg.corr_nan_policy,
             on_empty_stage=cfg.on_empty_stage,
             prefit_woe_engine=prefit_woe_engine,
+            missing_rate_threshold=cfg.missing_rate_threshold,
+            missing_rate_ref=cfg.missing_rate_ref,
         )
 
     if use_woe_bins or prefit_woe_engine is not None:
@@ -515,6 +543,8 @@ def feature_screen(
         plot_outputs=cfg.plot_outputs,
         iv_equal_freq=cfg.iv_equal_freq,
         on_empty_stage=cfg.on_empty_stage,
+        missing_rate_threshold=cfg.missing_rate_threshold,
+        missing_rate_ref=cfg.missing_rate_ref,
     )
 
 

{supermodelingfactory-0.3.14 → supermodelingfactory-0.4.2}/Modeling_Tool/Feature/PSI_Tool.py

@@ -7,12 +7,20 @@ to measure the distribution drift between expected and actual datasets.
 Author: Matrix Agent
 """
 
+import warnings
+
 import numpy as np
 import pandas as pd
 from typing import Union, List, Dict, Optional, Tuple, Callable, Any
 from tqdm import tqdm
 from Modeling_Tool.Core.Binning_Tool import quick_binning
 
+# Missing bucket label used across PSI helpers when ``missing_policy="include"``
+# (the 0.4.2 default). Kept identical to the sentinel already used by
+# ``Feature.Weighted_Screen`` and ``WOE_Adapter`` so downstream aggregators can
+# recognise it without special-casing.
+_MISSING_BIN = "__MISSING__"
+
 # ============================================================================
 # Classes
 # ============================================================================
@@ -460,14 +468,29 @@ def _calculate_single_psi(
     return_details: bool = False,
     min_bin_prop: float = 0.05,
     content: float = 1e-6,
-    precision: int = 5
+    precision: int = 5,
+    missing_policy: str = "drop",
 ) -> Union[float, Tuple[float, pd.DataFrame]]:
     """
     Calculate Population Stability Index (PSI) for a single variable.
-    
+
     This function bins both expected and actual series using the same breakpoints,
     then calculates PSI based on the distribution difference between them.
-    
+
+    Missing handling (0.4.2, N29)
+    ------------------------------
+    Before 0.4.2, both series were unconditionally passed through ``.dropna()``
+    before binning. Any drift in the *missing rate itself* — the single most
+    common early signal of a data-pipeline break — was silently invisible in
+    the returned PSI. 0.4.2 adds the ``missing_policy`` parameter so callers
+    can opt in to the corrected behaviour. The default remains ``"drop"`` for
+    strict backward compatibility with the pre-0.4.2 numeric output; the next
+    minor release will flip the default to ``"include"``. Pass
+    ``missing_policy="include"`` to route NaN rows through a dedicated
+    ``"__MISSING__"`` bin on both sides so missing-rate drift contributes to
+    the PSI. Pass ``missing_policy="warn_and_drop"`` to keep the old numbers
+    but at least surface a ``RuntimeWarning`` naming the two NaN counts.
+
     Parameters
     ----------
     expected_series : pandas.Series
@@ -486,62 +509,112 @@ def _calculate_single_psi(
         Small value to avoid division by zero. Default is 1e-6.
     precision : int, optional
         Decimal precision. Default is 5.
-        
+    missing_policy : {"drop", "include", "warn_and_drop"}, optional
+        How NaN rows are handled. ``"drop"`` (default in 0.4.2) reproduces the
+        pre-0.4.2 behaviour and silently excludes NaN rows from both sides
+        before binning. ``"include"`` treats NaN as its own ``"__MISSING__"``
+        bin so missing-rate drift shows up in the PSI — recommended for any
+        production drift monitor, and will become the default in the next
+        minor release. ``"warn_and_drop"`` behaves like ``"drop"`` but emits
+        a ``RuntimeWarning`` naming the two NaN counts so the caller sees what
+        was dropped.
+
     Returns
     -------
     float or tuple
         If return_details is False: Returns total PSI value.
         If return_details is True: Returns tuple of (PSI value, details DataFrame).
-        
+
     Notes
     -----
     PSI Formula: Σ (Actual% - Expected%) * ln(Actual% / Expected%)
     A PSI < 0.1 indicates stable population, 0.1-0.25 suggests some change,
     and > 0.25 indicates significant drift.
     """
-    # Drop NA
+    if missing_policy not in {"include", "drop", "warn_and_drop"}:
+        raise ValueError(
+            f"_calculate_single_psi: missing_policy must be one of "
+            f"'include', 'drop', 'warn_and_drop'; got {missing_policy!r}."
+        )
+
+    # Record NaN counts before splitting so we can (a) re-attach the missing
+    # bin under "include" and (b) warn under "warn_and_drop".
+    n_expected = int(len(expected_series))
+    n_actual = int(len(actual_series))
+    n_expected_na = int(expected_series.isna().sum())
+    n_actual_na = int(actual_series.isna().sum())
+
+    if missing_policy == "warn_and_drop" and (n_expected_na or n_actual_na):
+        warnings.warn(
+            f"_calculate_single_psi: dropping NaN rows before binning. "
+            f"expected: {n_expected_na}/{n_expected} NaN, "
+            f"actual: {n_actual_na}/{n_actual} NaN. "
+            f"Pass missing_policy='include' to treat NaN as its own bin so "
+            f"missing-rate drift is captured.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+
     expected_clean = expected_series.dropna()
     actual_clean = actual_series.dropna()
-    
+
     # Bin the expected data to get breakpoints
     expected_bins, breakpoints = quick_binning(
-        pd.DataFrame(expected_clean), 
-        expected_clean.name, 
-        labels=None, 
-        nbins=buckets, 
-        precision=precision, 
-        equal_freq=equal_freq, 
-        right=True, 
+        pd.DataFrame(expected_clean),
+        expected_clean.name,
+        labels=None,
+        nbins=buckets,
+        precision=precision,
+        equal_freq=equal_freq,
+        right=True,
         include_lowest=False,
-        min_bin_prop=min_bin_prop, 
-        tree_binning=False, 
-        target=None, 
+        min_bin_prop=min_bin_prop,
+        tree_binning=False,
+        target=None,
         random_state=42
     )
-    
+
     # Bin the actual data using the same breakpoints
     actual_bins, _ = quick_binning(
-        pd.DataFrame(actual_clean), 
-        actual_clean.name, 
-        labels=None, 
-        nbins=list(breakpoints), 
-        precision=precision, 
-        equal_freq=equal_freq, 
-        right=True, 
+        pd.DataFrame(actual_clean),
+        actual_clean.name,
+        labels=None,
+        nbins=list(breakpoints),
+        precision=precision,
+        equal_freq=equal_freq,
+        right=True,
         include_lowest=False,
-        min_bin_prop=min_bin_prop, 
-        tree_binning=False, 
-        target=None, 
+        min_bin_prop=min_bin_prop,
+        tree_binning=False,
+        target=None,
         random_state=42
     )
-    
-    # Get bin counts
+
+    # Get bin counts (finite-value rows only, from quick_binning)
     expected_count = expected_bins.value_counts(normalize=False, sort=False)
     actual_count = actual_bins.value_counts(normalize=False, sort=False)
-    
-    # Get bin proportions
-    expected_percents = expected_bins.value_counts(normalize=True, sort=False)
-    actual_percents = actual_bins.value_counts(normalize=True, sort=False)
+
+    if missing_policy == "include":
+        # Re-attach the __MISSING__ bin so that missing-rate drift contributes
+        # to the PSI on both sides. The denominators use the *original* row
+        # counts (before dropna) so the fractions are directly comparable.
+        if n_expected_na > 0:
+            expected_count = pd.concat(
+                [expected_count, pd.Series({_MISSING_BIN: n_expected_na})]
+            )
+        if n_actual_na > 0:
+            actual_count = pd.concat(
+                [actual_count, pd.Series({_MISSING_BIN: n_actual_na})]
+            )
+        expected_denom = float(n_expected) if n_expected > 0 else 1.0
+        actual_denom = float(n_actual) if n_actual > 0 else 1.0
+        expected_percents = expected_count / expected_denom
+        actual_percents = actual_count / actual_denom
+    else:
+        # "drop" / "warn_and_drop": legacy behaviour, denominators are the
+        # NaN-dropped row counts, so missing-rate drift is invisible.
+        expected_percents = expected_bins.value_counts(normalize=True, sort=False)
+        actual_percents = actual_bins.value_counts(normalize=True, sort=False)
     
     # Ensure both series have the same bin indices
     all_bins = expected_percents.index.union(actual_percents.index)

{supermodelingfactory-0.3.14 → supermodelingfactory-0.4.2}/Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py

@@ -19,7 +19,7 @@ from .Feature_Insights import (
     VarExtractionInsights as _BaseVarExtractionInsights,
     var_corr_filter,
 )
-from .Distribution_Tool import proc_means_by_grp
+from .Distribution_Tool import proc_means_for_screening
 
 
 def _psi_from_bins(expected_bins: pd.Series, current_bins: pd.Series, content: float) -> float:
@@ -110,9 +110,9 @@ def _screening_summary_from_engine(
     high_iv_summary = pd.DataFrame(rows).query(f"iv >= {iv_cut}").round(4)
     high_iv_varlist = high_iv_summary["var"].tolist()
     if high_iv_varlist:
-        means = proc_means_by_grp(data, high_iv_varlist, spec_missing_value=missing_rate_ref)
+        means = proc_means_for_screening(data, high_iv_varlist, spec_missing_value=missing_rate_ref)
         summary = high_iv_summary.merge(
-            means[["attribute", "N_ALL", "N", "MISSING_RATE", "MIN", "MEAN", "MAX"]],
+            means,
             left_on="var",
             right_on="attribute",
             how="left",