SuperModelingFactory 0.5.2__tar.gz → 0.5.4__tar.gz

{supermodelingfactory-0.5.2 → supermodelingfactory-0.5.4}/Modeling_Tool/Core/Binning_Tool.py

@@ -367,8 +367,7 @@ class NumVarBinning:
         if modify:
             self.cut_points = self.modify_cutpoints(df, points)
         else:
-            points.sort()
-            self.cut_points = points
+            self.cut_points = sorted(points)
         self.bins = len(self.cut_points) - 1
 
         binning_series = pd.cut(df[self.var_name], self.cut_points, right=False)

{supermodelingfactory-0.5.2 → supermodelingfactory-0.5.4}/Modeling_Tool/Core/ODPS_Tool.py

@@ -39,6 +39,41 @@ def _make_related_odps_table_name(table_name, suffix):
     return related_bare, related_bare
 
 
+def _parse_odps_partition_spec(partition):
+    if isinstance(partition, dict):
+        return [(str(k).strip(), str(v).strip().strip("'\"")) for k, v in partition.items()]
+    pairs = []
+    for item in str(partition).split(","):
+        if not item.strip():
+            continue
+        if "=" not in item:
+            raise ValueError(f"Invalid ODPS partition spec item {item!r}; expected key=value.")
+        key, value = item.split("=", 1)
+        pairs.append((key.strip(), value.strip().strip("'\"")))
+    if not pairs:
+        raise ValueError("partition must contain at least one key=value pair.")
+    return pairs
+
+
+def _format_odps_partition_spec(pairs, quoted=False):
+    parts = []
+    for key, value in pairs:
+        safe_value = str(value).replace("'", "''")
+        if quoted:
+            parts.append(f"{key}='{safe_value}'")
+        else:
+            parts.append(f"{key}={safe_value}")
+    return ",".join(parts)
+
+
+def _make_related_odps_partition_spec(partition, suffix):
+    pairs = _parse_odps_partition_spec(partition)
+    related = list(pairs)
+    key, value = related[-1]
+    related[-1] = (key, f"{value}{suffix}")
+    return _format_odps_partition_spec(related, quoted=False)
+
+
 class ODPSRunner(object):
     _wide_schema_patch_lock = threading.RLock()
     _wide_schema_patch_ref_count = 0
@@ -392,7 +427,21 @@ class ODPSRunner(object):
 
         logger.info(f'<<<< 完成数据入表{table_name}: shape={df.shape} >>>>')
 
-    def insert_df(self, df, table_name, overwrite=True, partition=None):
+    def _partition_exists(self, table, partition):
+        if hasattr(table, "exist_partition"):
+            return bool(table.exist_partition(partition))
+        try:
+            table.get_partition(partition)
+            return True
+        except Exception:
+            return False
+
+    def _rename_partition(self, table_name, source_partition, target_partition):
+        src = _format_odps_partition_spec(_parse_odps_partition_spec(source_partition), quoted=True)
+        dst = _format_odps_partition_spec(_parse_odps_partition_spec(target_partition), quoted=True)
+        self.o.run_sql(f"ALTER TABLE {table_name} PARTITION ({src}) RENAME TO PARTITION ({dst});")
+
+    def insert_df(self, df, table_name, overwrite=True, partition=None, atomic=True):
         """将数据集插入至mc已存在的表中.
 
         Parameters
@@ -411,6 +460,53 @@ class ODPSRunner(object):
             df.loc[:, "py_inserttime"] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
 
         if bool(partition):
+            if overwrite and atomic:
+                ts = datetime.now().strftime('%Y%m%d%H%M%S%f')
+                staging_partition = _make_related_odps_partition_spec(partition, f"__staging_{ts}")
+                backup_partition = _make_related_odps_partition_spec(partition, f"__old_{ts}")
+                target_exists = self._partition_exists(t, partition)
+
+                for stale in (staging_partition, backup_partition):
+                    try:
+                        t.delete_partition(stale, if_exists=True)
+                    except Exception as cleanup_exc:  # noqa: BLE001
+                        logger.warning(
+                            f"insert_df: failed to cleanup partition {stale} before atomic write: {cleanup_exc!r}"
+                        )
+
+                try:
+                    with t.open_writer(partition=staging_partition, create_partition=True) as writer:
+                        writer.write(df.values.tolist())
+                    if target_exists:
+                        self._rename_partition(table_name, partition, backup_partition)
+                    self._rename_partition(table_name, staging_partition, partition)
+                except Exception:
+                    if target_exists:
+                        try:
+                            if not self._partition_exists(t, partition) and self._partition_exists(t, backup_partition):
+                                self._rename_partition(table_name, backup_partition, partition)
+                        except Exception as restore_exc:  # noqa: BLE001
+                            logger.error(
+                                f"insert_df: failed to restore original partition {partition} "
+                                f"from {backup_partition}: {restore_exc!r}. Manual recovery may be required."
+                            )
+                    try:
+                        t.delete_partition(staging_partition, if_exists=True)
+                    except Exception:  # noqa: BLE001
+                        pass
+                    raise
+
+                if target_exists:
+                    try:
+                        t.delete_partition(backup_partition, if_exists=True)
+                    except Exception as cleanup_exc:  # noqa: BLE001
+                        logger.warning(
+                            f"insert_df: atomic swap succeeded but failed to drop backup partition "
+                            f"{backup_partition}: {cleanup_exc!r}. Safe to delete manually."
+                        )
+                logger.info('<<<< insert_df atomic partition write done: shape={0} >>>>'.format(df.shape))
+                return
+
             if overwrite:
                 t.delete_partition(partition, if_exists=True)
             with t.open_writer(partition=partition, create_partition=True) as writer:

{supermodelingfactory-0.5.2 → supermodelingfactory-0.5.4}/Modeling_Tool/Core/sample_weight_utils.py

@@ -52,6 +52,12 @@ def validate_sample_weight(weights, expected_len=None):
         raise ValueError("sample_weight must be finite (no NaN/inf).")
     if np.any(w < 0):
         raise ValueError("sample_weight must be non-negative.")
+    total = float(np.sum(w))
+    if total <= 0:
+        raise ValueError(
+            "sample_weight sum must be > 0; got sum={0}. "
+            "All-zero weights make weighted metrics undefined.".format(total)
+        )
     return w
 
 

{supermodelingfactory-0.5.2 → supermodelingfactory-0.5.4}/Modeling_Tool/Eval/Model_Eval_Tool.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import warnings
 import numpy as np
 import pandas as pd
 from Modeling_Tool.Core.Binning_Tool import get_bin_range_list, super_binning
@@ -2246,8 +2247,9 @@ class PerformanceEvaluator:
         self.weight_col = weight_col
         self.datasets = {}
         self.dataset_weight_cols = {}
+        self.evaluate_status = None
     
-    def add_dataset(self, name, data, weight_col = None):
+    def add_dataset(self, name, data, weight_col = None, overwrite = False):
         """
         添加数据集。
         
@@ -2263,6 +2265,17 @@ class PerformanceEvaluator:
         self
             返回自身以便链式调用
         """
+        if name in self.datasets and not overwrite:
+            raise KeyError(
+                f"Dataset name {name!r} already exists. Use overwrite=True to replace it, "
+                "or pass a different dataset name."
+            )
+        if name in self.datasets and overwrite:
+            warnings.warn(
+                f"PerformanceEvaluator.add_dataset overwriting existing dataset {name!r}.",
+                RuntimeWarning,
+                stacklevel=2,
+            )
         self.datasets[name] = data
         self.dataset_weight_cols[name] = weight_col
         return self
@@ -2304,16 +2317,38 @@ class PerformanceEvaluator:
             ``fig_save_path`` 自动按标签加后缀 (如 ``perf.png`` → ``perf_<label>.png``)。
         """
         if len(self.datasets) == 0:
+            self.evaluate_status = "no_datasets"
             return pd.DataFrame([])
 
         if self.scr_name is None and self.model is None and self.feature_cols is None:
-            return -1
+            self.evaluate_status = "compute_failed"
+            warnings.warn(
+                "PerformanceEvaluator.evaluate() now returns an empty DataFrame instead of -1; "
+                "inspect .evaluate_status for the failure reason.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            return pd.DataFrame([])
         
         if self.scr_name is None and self.model is None:
-            return -2
+            self.evaluate_status = "compute_failed"
+            warnings.warn(
+                "PerformanceEvaluator.evaluate() now returns an empty DataFrame instead of -2; "
+                "inspect .evaluate_status for the failure reason.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            return pd.DataFrame([])
         
         if self.scr_name is None and self.feature_cols is None:
-            return -3
+            self.evaluate_status = "compute_failed"
+            warnings.warn(
+                "PerformanceEvaluator.evaluate() now returns an empty DataFrame instead of -3; "
+                "inspect .evaluate_status for the failure reason.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            return pd.DataFrame([])
 
         active_weight_col = weight_col or self.weight_col
         has_dataset_weight = any(v is not None for v in self.dataset_weight_cols.values())
@@ -2369,6 +2404,7 @@ class PerformanceEvaluator:
                     )
             if rpt_save_path:
                 fnl_df.to_csv(rpt_save_path, index=False)
+            self.evaluate_status = "ok" if not fnl_df.empty else "empty_input"
             return fnl_df
 
         # ── 多 y 标签支持: tgt_name 为 list/tuple 时, 逐标签评估后纵向拼接 ──
@@ -2412,6 +2448,7 @@ class PerformanceEvaluator:
                 _ipy_display(fnl_df)
             if rpt_save_path:
                 fnl_df.to_csv(rpt_save_path, index = False)
+            self.evaluate_status = "ok" if not fnl_df.empty else "empty_input"
             return fnl_df
 
         def _get_score(data):
@@ -2472,6 +2509,7 @@ class PerformanceEvaluator:
                 }
 
             if len(eval_datasets) == 0:
+                self.evaluate_status = "empty_input"
                 return pd.DataFrame([])
 
             model_eval_result_df = evaluate_performance(
@@ -2570,4 +2608,5 @@ class PerformanceEvaluator:
         if rpt_save_path:
             fnl_df.to_csv(rpt_save_path, index = False)
 
+        self.evaluate_status = "ok" if isinstance(fnl_df, pd.DataFrame) and not fnl_df.empty else "empty_input"
         return fnl_df

{supermodelingfactory-0.5.2 → supermodelingfactory-0.5.4}/Modeling_Tool/Eval/Model_Eval_Tool.pyi

@@ -32,6 +32,6 @@ class GainsTableCalculator:
     def calculate(self, grp_name = None, min_data_size = 100, grp_colname = None, sync_range = True, retSummary = False, withSummary = False, wholeGroup = False, add_func = None): ...
 
 class PerformanceEvaluator:
-    def __init__(self, tgt_name, scr_name = None, model = None, feature_cols = None, dist_bins = 20, pct_bins = 10, precision = 5, min_bin_prop = 0.05, include_missing = False, equal_freq = True, chi2_method = False, init_equi_bins = 1000, chi2_p = 0.9, tree_binning = False, random_state = 42): ...
-    def add_dataset(self, name, data): ...
-    def evaluate(self, oot_grp_name = None, min_data_size = 100, grp_colname = None, fig_save_path = None, rpt_save_path = None, to_show = False, display = True, gains_table = False, benchmark_dataset = None): ...
+    def __init__(self, tgt_name, scr_name = None, model = None, feature_cols = None, dist_bins = 20, pct_bins = 10, precision = 5, min_bin_prop = 0.05, include_missing = False, equal_freq = True, chi2_method = False, init_equi_bins = 1000, chi2_p = 0.9, tree_binning = False, random_state = 42, weight_col = None): ...
+    def add_dataset(self, name, data, weight_col = None, overwrite = False): ...
+    def evaluate(self, oot_grp_name = None, min_data_size = 100, grp_colname = None, fig_save_path = None, rpt_save_path = None, to_show = False, display = True, gains_table = False, benchmark_dataset = None, weight_col = None): ...

{supermodelingfactory-0.5.2 → supermodelingfactory-0.5.4}/Modeling_Tool/Eval/weighted_eval_utils.py

@@ -14,6 +14,7 @@ import pandas as pd
 from sklearn.metrics import precision_recall_curve, roc_auc_score, roc_curve
 
 from Modeling_Tool.Core.sample_weight_utils import resolve_sample_weight
+from Modeling_Tool._utils.robust import smf_logger
 
 
 def resolve_weights(data=None, weight_col=None, sample_weight=None, expected_len=None, wgt=None, wgt_col=None):
@@ -43,7 +44,8 @@ def safe_weighted_average(values, weights=None):
 def safe_auc(y_true, y_score, sample_weight=None):
     try:
         return float(roc_auc_score(y_true, y_score, sample_weight=sample_weight))
-    except Exception:
+    except (ValueError, ZeroDivisionError) as exc:
+        smf_logger.record_and_continue("auc", exc, stage="weighted_eval.safe_auc")
         return np.nan
 
 

{supermodelingfactory-0.5.2 → supermodelingfactory-0.5.4}/Modeling_Tool/Model/LRM_Tool.py

@@ -6,6 +6,7 @@ import pandas as pd
 from sklearn.linear_model import LogisticRegression
 import logging
 from Modeling_Tool.Core.sample_weight_utils import resolve_sample_weight
+from Modeling_Tool._utils.nan_guard import warn_if_nan_ratio_exceeds
 
 logger = logging.getLogger(__name__)
 
@@ -252,6 +253,33 @@ def compute_bic(model, x, y, sample_weight=None):
     return bic
 
 
+def _prepare_nan_handled_frame(data, nan_handling="fillna_median", warn_threshold=0.05, context=""):
+    """Return a numeric frame with explicit NaN handling for selection stats."""
+    allowed = {"fillna_0", "fillna_mean", "fillna_median", "drop_rows", "raise"}
+    if nan_handling not in allowed:
+        raise ValueError(f"nan_handling must be one of {sorted(allowed)}; got {nan_handling!r}")
+
+    frame = data.copy()
+    for col in frame.columns:
+        warn_if_nan_ratio_exceeds(frame[col], threshold=warn_threshold, context=f"{context}.{col}")
+
+    if nan_handling == "raise":
+        if frame.isna().any().any():
+            missing_cols = frame.columns[frame.isna().any()].tolist()
+            raise ValueError(f"{context}: NaN values present in columns {missing_cols}")
+        return frame
+    if nan_handling == "drop_rows":
+        return frame.dropna(axis=0)
+    if nan_handling == "fillna_0":
+        return frame.fillna(0)
+
+    if nan_handling == "fillna_mean":
+        fill_values = frame.mean(numeric_only=True).fillna(0)
+    else:
+        fill_values = frame.median(numeric_only=True).fillna(0)
+    return frame.fillna(fill_values).fillna(0)
+
+
 class FeatureSelectionAnalyzer:
     """
     Feature selection analyzer using statistical tests.
@@ -284,7 +312,7 @@ class FeatureSelectionAnalyzer:
         self.selected_features_ = None
         self.chi2_results_ = None
 
-    def chi2_selection(self, data, feature_cols, target_col):
+    def chi2_selection(self, data, feature_cols, target_col, nan_handling="fillna_median", nan_warn_threshold=0.05):
         """
         Select features using chi-squared test.
 
@@ -305,8 +333,13 @@ class FeatureSelectionAnalyzer:
         from sklearn.feature_selection import chi2
         from sklearn.preprocessing import MinMaxScaler
 
-        x = data[feature_cols].fillna(0)
-        y = data[target_col]
+        x = _prepare_nan_handled_frame(
+            data[feature_cols],
+            nan_handling=nan_handling,
+            warn_threshold=nan_warn_threshold,
+            context="FeatureSelectionAnalyzer.chi2_selection",
+        )
+        y = data.loc[x.index, target_col]
 
         scaler = MinMaxScaler()
         x_scaled = scaler.fit_transform(x)
@@ -324,7 +357,7 @@ class FeatureSelectionAnalyzer:
         self.selected_features_ = results.loc[results['selected'], 'feature'].tolist()
         return results
 
-    def compute_vif(self, data):
+    def compute_vif(self, data, nan_handling="fillna_median", nan_warn_threshold=0.05):
         """
         Compute Variance Inflation Factor (VIF) for multicollinearity detection.
 
@@ -340,9 +373,15 @@ class FeatureSelectionAnalyzer:
         """
         from statsmodels.stats.outliers_influence import variance_inflation_factor
 
-        x = data.fillna(0).values
+        work = _prepare_nan_handled_frame(
+            data,
+            nan_handling=nan_handling,
+            warn_threshold=nan_warn_threshold,
+            context="FeatureSelectionAnalyzer.compute_vif",
+        )
+        x = work.values
         vif_data = pd.DataFrame({
-            'feature': data.columns,
+            'feature': work.columns,
             'VIF': [variance_inflation_factor(x, i) for i in range(x.shape[1])]
         }).sort_values('VIF', ascending=False).reset_index(drop=True)
 

{supermodelingfactory-0.5.2 → supermodelingfactory-0.5.4}/Modeling_Tool/Model/LRM_Tool.pyi

@@ -26,8 +26,8 @@ def compute_bic(model, x, y): ...
 
 class FeatureSelectionAnalyzer:
     def __init__(self, significance_level = 0.05): ...
-    def chi2_selection(self, data, feature_cols, target_col): ...
-    def compute_vif(self, data): ...
+    def chi2_selection(self, data, feature_cols, target_col, nan_handling = "fillna_median", nan_warn_threshold = 0.05): ...
+    def compute_vif(self, data, nan_handling = "fillna_median", nan_warn_threshold = 0.05): ...
     def correlation_filter(self, data, threshold = 0.8): ...
 
 class LRMaster:

{supermodelingfactory-0.5.2 → supermodelingfactory-0.5.4}/Modeling_Tool/Pipeline/__init__.py

@@ -31,6 +31,23 @@ from .feature_validation import (
 )
 from .orchestrator import run_modeling_from_validation
 from .screening_artifact import FeatureScreeningArtifact, screen_result_to_summary
+from .field_meta import (
+    FieldMeta,
+    PipelineRegistryEntry,
+    PIPELINE_REGISTRY,
+    config_from_dict,
+    config_from_yaml,
+    config_to_dict,
+    config_to_yaml,
+    extract_config_schema,
+    extract_pipeline_schema,
+    extract_schema,
+    generate_pipeline_code,
+    get_config_field_meta,
+    get_pipeline_registry,
+    get_pipeline_registry_schema,
+    validate_pipeline_config,
+)
 
 __all__ = [
     "RejectInferencePipeline",
@@ -57,4 +74,19 @@ __all__ = [
     "FeatureScreeningArtifact",
     "screen_result_to_summary",
     "run_modeling_from_validation",
+    "FieldMeta",
+    "PipelineRegistryEntry",
+    "PIPELINE_REGISTRY",
+    "get_pipeline_registry",
+    "get_pipeline_registry_schema",
+    "get_config_field_meta",
+    "extract_config_schema",
+    "extract_pipeline_schema",
+    "extract_schema",
+    "config_to_dict",
+    "config_from_dict",
+    "config_to_yaml",
+    "config_from_yaml",
+    "validate_pipeline_config",
+    "generate_pipeline_code",
 ]

{supermodelingfactory-0.5.2 → supermodelingfactory-0.5.4}/Modeling_Tool/Pipeline/_common.py

@@ -285,7 +285,12 @@ def get_raw_model(model: Any) -> Any:
     return model
 
 
-def predict_positive(model: Any, data: pd.DataFrame, feature_cols: list[str]) -> np.ndarray:
+def predict_positive(
+    model: Any,
+    data: pd.DataFrame,
+    feature_cols: list[str],
+    warn_nan: bool = True,
+) -> np.ndarray:
     raw_model = get_raw_model(model)
     x = data[feature_cols]
 
@@ -338,7 +343,7 @@ def predict_positive(model: Any, data: pd.DataFrame, feature_cols: list[str]) ->
         )
 
     finite_mask = np.isfinite(result)
-    if not finite_mask.all():
+    if warn_nan and not finite_mask.all():
         n_bad = int((~finite_mask).sum())
         warnings.warn(
             f"predict_positive: {n_bad}/{n_rows} predictions are NaN/Inf "
@@ -349,6 +354,35 @@ def predict_positive(model: Any, data: pd.DataFrame, feature_cols: list[str]) ->
     return result
 
 
+def predict_positive_many(
+    model: Any,
+    datasets: Mapping[str, pd.DataFrame],
+    feature_cols: list[str],
+) -> tuple[dict[str, np.ndarray], dict[str, int]]:
+    """Predict multiple datasets and emit one aggregate NaN/Inf warning."""
+    preds: dict[str, np.ndarray] = {}
+    stats: dict[str, int] = {}
+    total_bad = 0
+    total_rows = 0
+    for name, data in datasets.items():
+        pred = predict_positive(model, data, feature_cols, warn_nan=False)
+        bad = int((~np.isfinite(pred)).sum())
+        stats[str(name)] = bad
+        total_bad += bad
+        total_rows += len(pred)
+        preds[str(name)] = pred
+
+    if total_bad:
+        detail = ", ".join(f"{name}={count}" for name, count in stats.items() if count)
+        warnings.warn(
+            f"NaN/Inf predictions coerced or returned: {detail}; "
+            f"total {total_bad}/{total_rows}.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+    return preds, stats
+
+
 def add_dataset_with_optional_weight(
     evaluator: Any,
     name: str,

{supermodelingfactory-0.5.2 → supermodelingfactory-0.5.4}/Modeling_Tool/Pipeline/credit_model.py

@@ -6,6 +6,7 @@ from typing import Any, Literal
 
 import numpy as np
 import pandas as pd
+import warnings
 
 from ._common import (
     add_dataset_with_optional_weight,
@@ -175,6 +176,7 @@ class CreditModelPipeline:
 
     def __init__(self, config: CreditModelPipelineConfig | None = None):
         self.config = config or CreditModelPipelineConfig()
+        self.predict_positive_nan_stats: dict[str, dict[str, int]] = {}
         self._validate_gbm_feature_source_config()
 
     def run(self, data: pd.DataFrame) -> CreditModelPipelineResult:
@@ -1101,10 +1103,23 @@ class CreditModelPipeline:
                 equal_freq=True,
             )
             eval_splits = {**splits, **model_extra}
+            nan_stats: dict[str, int] = {}
             for ds_name, df in eval_splits.items():
                 scored = df.copy()
                 scored[f"pred_{name}"] = self._predict_model_positive(name, wrapper, scored, feature_cols)
+                nan_stats[str(ds_name)] = int((~np.isfinite(scored[f"pred_{name}"].to_numpy(dtype=float))).sum())
                 add_dataset_with_optional_weight(evaluator, ds_name, scored, weight_col=cfg.weight_col)
+            self.predict_positive_nan_stats[str(name)] = nan_stats
+            total_bad = sum(nan_stats.values())
+            if total_bad:
+                total_rows = sum(len(df) for df in eval_splits.values())
+                detail = ", ".join(f"{k}={v}" for k, v in nan_stats.items() if v)
+                warnings.warn(
+                    f"{name}: NaN/Inf predictions detected across evaluation datasets: "
+                    f"{detail}; total {total_bad}/{total_rows}.",
+                    RuntimeWarning,
+                    stacklevel=2,
+                )
             fig_save_path = None
             if cfg.write_outputs and cfg.plot_outputs:
                 fig_save_path = str(Path(cfg.output_dir) / "figs" / "perf" / f"perf_{name}.png")
@@ -1121,7 +1136,7 @@ class CreditModelPipeline:
         if self._warm_start_requested_for(model_name) and model_name in {"lgb", "xgb"}:
             init_score = self._get_warm_start_init_score(model_name, data)
             return wrapper.predict_with_base_margin(data[feature_cols], init_score, return_prob=True)
-        return predict_positive(wrapper, data, feature_cols)
+        return predict_positive(wrapper, data, feature_cols, warn_nan=False)
 
     def _will_run_explainability(self) -> bool:
         cfg = self.config