SuperModelingFactory 0.3.2__tar.gz → 0.3.4__tar.gz

{supermodelingfactory-0.3.2 → supermodelingfactory-0.3.4}/Modeling_Tool/Eval/Evaluation_Tool.py

@@ -597,6 +597,7 @@ class Model_Evaluation_Tool:
         min_bin_prop: Optional[float] = None,
         include_missing: Optional[bool] = None,
         equal_freq: Optional[bool] = None,
+        sample_name: Optional[str] = None,
     ) -> pd.DataFrame:
         """
         Compare model performance across different scores.
@@ -626,7 +627,12 @@ class Model_Evaluation_Tool:
             Defaults to ``False`` for performance comparison.
         equal_freq : bool, optional
             Use equal-frequency binning. Defaults to the instance setting.
-            
+        sample_name : str, optional
+            Label applied to the evaluated dataset and emitted in the result's
+            ``index`` column. Defaults to ``'all'`` (the whole ``data`` treated as
+            a single sample). Callers evaluating a specific slice can pass e.g.
+            ``'global'`` or a split name.
+
         Returns
         -------
         pandas.DataFrame
@@ -636,7 +642,10 @@ class Model_Evaluation_Tool:
         score_list = self.comp_scrlist
         dep = self.dep
         base_score = self.base_score
-        sample_name = 'oot'
+        # Label for the single (unsplit) dataset being evaluated; surfaces as the
+        # 'index' column of the result. Defaults to a neutral 'all' rather than the
+        # historical (and misleading) 'oot', which implied an out-of-time sample.
+        sample_name = sample_name if sample_name is not None else 'all'
 
         if base_score is None:
             return pd.DataFrame()

{supermodelingfactory-0.3.2 → supermodelingfactory-0.3.4}/Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py

@@ -155,17 +155,35 @@ class PSICalculator:
 
         detail = {}
         rows = []
-        groups = [(None, current_data)] if group_by is None else list(current_data.groupby(group_by))
+        resolved_group = group_name if group_name is not None else group_by
+        if resolved_group is None:
+            groups = [(None, current_data, {})]
+        else:
+            group_cols = [resolved_group] if isinstance(resolved_group, str) else list(resolved_group)
+            grouped_data = current_data.copy()
+            for col in group_cols:
+                if grouped_data[col].isna().sum() > 0:
+                    grouped_data[col] = grouped_data[col].fillna("__NULL__")
+            group_key = group_cols[0] if len(group_cols) == 1 else group_cols
+            groups = []
+            for grp_value, grp_df in grouped_data.groupby(group_key, dropna=False):
+                if len(group_cols) == 1:
+                    group_info = {group_cols[0]: grp_value}
+                else:
+                    value_tuple = grp_value if isinstance(grp_value, tuple) else (grp_value,)
+                    group_info = dict(zip(group_cols, value_tuple))
+                groups.append((grp_value, grp_df, group_info))
+
         for var in varlist:
             expected_bins = adapter.assign_bins(expected_df, var)
-            for grp_value, grp_df in groups:
+            for grp_value, grp_df, group_info in groups:
                 current_bins = adapter.assign_bins(grp_df, var)
                 row = {"var": var, "psi": round(_psi_from_bins(expected_bins, current_bins, self.content), self.precision)}
-                if group_by is not None:
-                    row[group_name or group_by] = grp_value
+                row.update(group_info)
                 rows.append(row)
                 if return_details:
-                    detail[var if grp_value is None else (var, grp_value)] = {
+                    detail_key = var if resolved_group is None else (var, grp_value)
+                    detail[detail_key] = {
                         "expected_bins": expected_bins.value_counts(normalize=True, dropna=False),
                         "current_bins": current_bins.value_counts(normalize=True, dropna=False),
                     }

{supermodelingfactory-0.3.2 → supermodelingfactory-0.3.4}/Modeling_Tool/Model/GBM_Tool.py

@@ -153,7 +153,9 @@ def _normalize_catboost_params(params_dict):
     ``eval_metric`` defaults to ``'AUC'`` when neither ``eval_metric`` nor
     ``metric`` is supplied. Common lower-case aliases (e.g. ``'auc'``,
     ``'logloss'``) are normalized to the case-sensitive names CatBoost expects;
-    any other string is left untouched.
+    any other string is left untouched. CatBoost writes ``catboost_info`` under
+    the current working directory by default, so SMF disables file output unless
+    the caller explicitly passes ``allow_writing_files`` or ``train_dir``.
 
     Returns
     -------
@@ -174,6 +176,8 @@ def _normalize_catboost_params(params_dict):
 
     params.setdefault('loss_function', 'Logloss')
     params.setdefault('verbose', False)
+    if 'allow_writing_files' not in params and 'train_dir' not in params:
+        params['allow_writing_files'] = False
 
     # eval_metric must be passed to the CatBoostClassifier constructor, not fit().
     if eval_metric is None:

{supermodelingfactory-0.3.2 → supermodelingfactory-0.3.4}/Modeling_Tool/Model/LRM_Tool.py

@@ -523,6 +523,44 @@ class LRMaster:
         self._data = data
         return self
 
+    def __getstate__(self):
+        """
+        Exclude the reference training frame from pickling.
+
+        ``self._data`` holds the full training set (row-level customer records)
+        only as a convenience fallback for
+        ``get_statsmodel_summary``/``get_aic``/``get_bic``. Serializing it bloats
+        saved models (tens to hundreds of MB) and leaks raw data alongside the
+        model artifact. Prediction never depends on it, so it is dropped on
+        pickle; pass ``data=`` explicitly to the diagnostic methods on a loaded
+        model.
+        """
+        state = self.__dict__.copy()
+        state["_data"] = None
+        return state
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+        self.__dict__.setdefault("_data", None)
+
+    def _require_data(self, data):
+        """
+        Resolve evaluation data, falling back to the stored training frame.
+
+        Raises a clear error when neither an explicit ``data`` argument nor a
+        stored ``self._data`` is available -- e.g. on a model loaded from disk,
+        where ``_data`` is intentionally dropped during pickling.
+        """
+        if data is None:
+            data = self._data
+        if data is None:
+            raise ValueError(
+                "No data available: this model was loaded without its training "
+                "frame (_data is dropped on save). Pass data=... to "
+                "get_statsmodel_summary/get_aic/get_bic."
+            )
+        return data
+
     def fit(self, data, varlist, tgt_name, val_data=None, val_varlist=None, val_tgt_name=None, weight_col=None):
         """
         Train the logistic regression model.
@@ -796,8 +834,7 @@ class LRMaster:
         When standardization is enabled the summary is computed on the
         standardized feature space, consistent with how the model was trained.
         """
-        if data is None:
-            data = self._data
+        data = self._require_data(data)
         if varlist is None:
             varlist = self.varlist
         if tgt_name is None:
@@ -824,8 +861,7 @@ class LRMaster:
         -------
         float
         """
-        if data is None:
-            data = self._data
+        data = self._require_data(data)
         if varlist is None:
             varlist = self.varlist
         if tgt_name is None:
@@ -851,8 +887,7 @@ class LRMaster:
         -------
         float
         """
-        if data is None:
-            data = self._data
+        data = self._require_data(data)
         if varlist is None:
             varlist = self.varlist
         if tgt_name is None:

{supermodelingfactory-0.3.2 → supermodelingfactory-0.3.4}/Modeling_Tool/Pipeline/credit_model.py

@@ -30,6 +30,10 @@ class CreditModelPipelineConfig:
     write_outputs: bool = True
     write_excel: bool = True
     plot_outputs: bool = True
+    save_models: bool = False
+    model_output_dir: str | None = None
+    model_include_metadata: bool = True
+    save_woe_artifacts: bool = True
 
     split_config: dict[str, Any] = field(default_factory=lambda: {"test_size": 0.3, "stratify": True})
     feature_selection: dict[str, Any] = field(
@@ -103,6 +107,8 @@ class CreditModelPipelineResult:
     model_feature_sets: dict[str, list[str]] = field(default_factory=dict)
     selected_raw_features: list[str] = field(default_factory=list)
     selected_woe_features: list[str] = field(default_factory=list)
+    model_paths: dict[str, str] = field(default_factory=dict)
+    artifact_paths: dict[str, str] = field(default_factory=dict)
 
 
 class CreditModelPipeline:
@@ -166,9 +172,10 @@ class CreditModelPipeline:
                 output_dir / "figs" / "woe",
                 output_dir / "figs" / "mono_woe",
                 output_dir / "figs" / "perf",
-                output_dir / "models",
                 output_dir / "explain",
             )
+        if cfg.save_models:
+            make_dirs(self._model_output_dir(), output_dir / "artifacts")
 
         splits = self._split_data(data)
         fs_summary, final_features = self._feature_selection(splits, feature_cols)
@@ -202,6 +209,19 @@ class CreditModelPipeline:
         optuna_results = self._run_optuna(model_inputs)
         perf_results = self._evaluate_models(model_inputs, models)
         explain_outputs = self._run_explainability(model_inputs, models)
+        model_paths: dict[str, str] = {}
+        artifact_paths: dict[str, str] = {}
+        model_paths_frame = None
+        if cfg.save_models:
+            model_paths, artifact_paths = self._save_models_and_artifacts(
+                models=models,
+                woe_artifacts=woe_artifacts,
+                perf_results=perf_results,
+                model_feature_sources=model_feature_sources,
+                model_feature_sets=model_feature_sets,
+            )
+            model_paths_frame = self._paths_to_frame(model_paths, artifact_paths)
+            safe_to_csv(model_paths_frame, output_dir / "model_paths.csv", index=False)
 
         if cfg.write_outputs:
             self._write_outputs(
@@ -214,6 +234,7 @@ class CreditModelPipeline:
                 lr_search_results,
                 warm_start_summary,
                 model_feature_source_summary,
+                model_paths_frame,
             )
 
         report_path = None
@@ -226,6 +247,7 @@ class CreditModelPipeline:
                 "LR_Param_Search": lr_search_results,
                 "Warm_Start": warm_start_summary,
                 "Model_Feature_Source": model_feature_source_summary,
+                "Model_Paths": model_paths_frame,
             }
             for name, perf in perf_results.items():
                 sheets[f"Perf_{name.upper()}"] = perf
@@ -248,6 +270,8 @@ class CreditModelPipeline:
             model_feature_sets=model_feature_sets,
             selected_raw_features=list(selected_raw_features),
             selected_woe_features=list(selected_woe_features),
+            model_paths=model_paths,
+            artifact_paths=artifact_paths,
         )
 
     def _resolve_feature_cols(self, data: pd.DataFrame) -> list[str]:
@@ -956,6 +980,92 @@ class CreditModelPipeline:
             },
         }
 
+    def _model_output_dir(self) -> Path:
+        cfg = self.config
+        return Path(cfg.model_output_dir) if cfg.model_output_dir else Path(cfg.output_dir) / "models"
+
+    def _save_models_and_artifacts(
+        self,
+        models: dict[str, tuple[Any, Any, list[str]]],
+        woe_artifacts: dict[str, Any],
+        perf_results: dict[str, pd.DataFrame],
+        model_feature_sources: dict[str, str],
+        model_feature_sets: dict[str, list[str]],
+    ) -> tuple[dict[str, str], dict[str, str]]:
+        from Modeling_Tool import save_model
+
+        cfg = self.config
+        model_dir = self._model_output_dir()
+        artifact_dir = Path(cfg.output_dir) / "artifacts"
+        make_dirs(model_dir)
+        model_paths: dict[str, str] = {}
+        artifact_paths: dict[str, str] = {}
+
+        woe_table_path = None
+        if cfg.save_woe_artifacts:
+            make_dirs(artifact_dir)
+            woe_table = woe_artifacts.get("woe_table")
+            if isinstance(woe_table, pd.DataFrame):
+                woe_table_path = artifact_dir / "woe_table.csv"
+                safe_to_csv(woe_table, woe_table_path, index=False)
+                artifact_paths["woe_table"] = str(woe_table_path)
+            engine = woe_artifacts.get("engine")
+            if engine is not None:
+                engine_path = artifact_dir / "woe_engine.pkl"
+                save_model(
+                    engine,
+                    engine_path,
+                    metadata={
+                        "pipeline": "CreditModelPipeline",
+                        "artifact_role": "woe_engine",
+                        "target_col": cfg.target_col,
+                        "raw_features": list(woe_artifacts.get("features") or []),
+                        "woe_features": list(woe_artifacts.get("woe_features") or []),
+                        "woe_engine": woe_artifacts.get("engine_name"),
+                        "random_state": cfg.random_state,
+                    },
+                    feature_cols=woe_artifacts.get("features"),
+                    include_metadata=cfg.model_include_metadata,
+                )
+                artifact_paths["woe_engine"] = str(engine_path)
+
+        warm_start_requested = {str(x).lower() for x in as_list(cfg.warm_start_models)}
+        for name, (wrapper, _, feature_cols) in models.items():
+            path = model_dir / f"model_{name}.pkl"
+            metadata = {
+                "pipeline": "CreditModelPipeline",
+                "model_name": name,
+                "target_col": cfg.target_col,
+                "feature_cols": list(feature_cols),
+                "feature_source": model_feature_sources.get(name),
+                "model_feature_set": model_feature_sets.get(name, list(feature_cols)),
+                "model_params": dict(cfg.model_params.get(name, {})),
+                "warm_start_enabled": bool(cfg.warm_start_enabled and name in warm_start_requested),
+                "warm_start_score_col": cfg.warm_start_score_col,
+                "random_state": cfg.random_state,
+            }
+            metrics = None
+            if isinstance(perf_results.get(name), pd.DataFrame):
+                metrics = {"perf_results": perf_results[name].to_dict("records")}
+            save_model(
+                wrapper,
+                path,
+                metadata=metadata,
+                feature_cols=feature_cols,
+                woe_mapping_path=str(woe_table_path) if woe_table_path else None,
+                metrics=metrics,
+                model_name=name,
+                include_metadata=cfg.model_include_metadata,
+            )
+            model_paths[name] = str(path)
+        return model_paths, artifact_paths
+
+    @staticmethod
+    def _paths_to_frame(model_paths: dict[str, str], artifact_paths: dict[str, str]) -> pd.DataFrame:
+        rows = [{"name": name, "path": path, "kind": "model"} for name, path in model_paths.items()]
+        rows.extend({"name": name, "path": path, "kind": "artifact"} for name, path in artifact_paths.items())
+        return pd.DataFrame(rows)
+
     def _write_outputs(
         self,
         output_dir: Path,
@@ -967,6 +1077,7 @@ class CreditModelPipeline:
         lr_search_results: pd.DataFrame | None,
         warm_start_summary: pd.DataFrame | None,
         model_feature_source_summary: pd.DataFrame | None,
+        model_paths_frame: pd.DataFrame | None,
     ) -> None:
         if isinstance(fs_summary.get("psi"), pd.DataFrame):
             safe_to_csv(fs_summary["psi"], output_dir / "psi_result.csv", index=False)
@@ -977,6 +1088,7 @@ class CreditModelPipeline:
         safe_to_csv(lr_search_results, output_dir / "lr_param_search.csv", index=False)
         safe_to_csv(warm_start_summary, output_dir / "warm_start_summary.csv", index=False)
         safe_to_csv(model_feature_source_summary, output_dir / "model_feature_sources.csv", index=False)
+        safe_to_csv(model_paths_frame, output_dir / "model_paths.csv", index=False)
         for name, df in optuna_results.items():
             safe_to_csv(df, output_dir / f"{name}_optuna_search.csv", index=False)
         for name, df in perf_results.items():

{supermodelingfactory-0.3.2 → supermodelingfactory-0.3.4}/Modeling_Tool/Pipeline/feature_validation.py

@@ -648,6 +648,7 @@ class FeatureValidationPipeline:
                     continue
                 params = dict(cfg.corr_params or {})
                 max_iterations = int(params.pop("max_iterations", 10))
+                corr_method = params.get("method", "pearson")
                 filt = CorrelationFilter(data=observed, dep=target, **params)
                 keep = filt.remove_highly_correlated(features, max_iterations=max_iterations)
                 removed = [feature for feature in features if feature not in keep]
@@ -659,6 +660,7 @@ class FeatureValidationPipeline:
                     for _, corr_row in corr.iterrows():
                         key = tuple(sorted([str(corr_row.get("VAR1")), str(corr_row.get("VAR2"))]))
                         meta = pair_meta.get(key, {})
+                        row_corr_method = meta.get("corr_method", corr_method)
                         for var in [corr_row.get("VAR1"), corr_row.get("VAR2")]:
                             metrics = metric_map.get(var, {})
                             rows.append(
@@ -673,9 +675,14 @@ class FeatureValidationPipeline:
                                     "iv": metrics.get("iv"),
                                     "ks_in_gains": metrics.get("ks_in_gains"),
                                     "lift_in_gains": metrics.get("lift_in_gains"),
+                                    **self._corr_detail_fields(
+                                        corr_row.get("VAR1"),
+                                        corr_row.get("VAR2"),
+                                        var,
+                                        row_corr_method,
+                                        "cross_batch",
+                                    ),
                                     "pair_type": "new_new_cross_batch",
-                                    "detail_scope": "cross_batch",
-                                    "corr_method": meta.get("corr_method", params.get("method", "pearson")),
                                     "batch_left": meta.get("batch_left"),
                                     "batch_right": meta.get("batch_right"),
                                 }
@@ -688,8 +695,14 @@ class FeatureValidationPipeline:
                             "corr_var1": features[0],
                             "corr_var2": features[1],
                             "recommended_action": "remove",
+                            **self._corr_detail_fields(
+                                features[0],
+                                features[1],
+                                ",".join(removed),
+                                pair.get("corr_method", params.get("method", "pearson")),
+                                "cross_batch",
+                            ),
                             "pair_type": "new_new_cross_batch",
-                            "detail_scope": "cross_batch",
                             "batch_left": pair.get("batch_left"),
                             "batch_right": pair.get("batch_right"),
                         }
@@ -1121,6 +1134,12 @@ class FeatureValidationPipeline:
                     result = pd.DataFrame(result)
                     if group_col is None and "psi" not in result.columns:
                         result = pd.DataFrame({"psi": result.iloc[:, 0]})
+                    if group_col is None:
+                        result["group_value"] = "global"
+                    elif group_col in result.columns:
+                        result["group_value"] = result[group_col]
+                    else:
+                        result["group_value"] = np.nan
                     result["target"] = target_key
                     result["group_col"] = group_col or "global"
                     rows.append(result)
@@ -1350,6 +1369,7 @@ class FeatureValidationPipeline:
                 continue
             params = dict(cfg.corr_params or {})
             max_iterations = int(params.pop("max_iterations", 10))
+            corr_method = params.get("method", "pearson")
             binner = woe_artifacts.get("by_target", {}).get(target, {}).get("engine") if cfg.corr_use_woe_bins else None
             filt = CorrelationFilter(data=data, dep=target, woe_binner=binner, **params)
             keep = filt.remove_highly_correlated(features, max_iterations=max_iterations)
@@ -1374,12 +1394,53 @@ class FeatureValidationPipeline:
                                 "iv": metrics.get("iv"),
                                 "ks_in_gains": metrics.get("ks_in_gains"),
                                 "lift_in_gains": metrics.get("lift_in_gains"),
+                                **self._corr_detail_fields(
+                                    row.get("VAR1"),
+                                    row.get("VAR2"),
+                                    var,
+                                    corr_method,
+                                    "within_batch",
+                                ),
                             }
                         )
             if not rows and removed:
-                rows.append({"target": target, "recommended_action": "remove", "var": ",".join(removed)})
+                rows.append(
+                    {
+                        "target": target,
+                        "recommended_action": "remove",
+                        "var": ",".join(removed),
+                        "metric_var": ",".join(removed),
+                        "corr_method": corr_method,
+                        "detail_scope": "within_batch",
+                    }
+                )
         return pd.DataFrame(rows)
 
+    @staticmethod
+    def _corr_detail_fields(
+        corr_var1: Any,
+        corr_var2: Any,
+        metric_var: Any,
+        corr_method: str,
+        detail_scope: str,
+    ) -> dict[str, Any]:
+        corr_var1_str = str(corr_var1) if pd.notna(corr_var1) else ""
+        corr_var2_str = str(corr_var2) if pd.notna(corr_var2) else ""
+        metric_var_str = str(metric_var) if pd.notna(metric_var) else ""
+        if metric_var_str == corr_var1_str:
+            metric_var_position = "corr_var1"
+        elif metric_var_str == corr_var2_str:
+            metric_var_position = "corr_var2"
+        else:
+            metric_var_position = np.nan
+        return {
+            "corr_pair_id": f"{corr_var1_str}||{corr_var2_str}" if corr_var1_str or corr_var2_str else np.nan,
+            "metric_var": metric_var,
+            "metric_var_position": metric_var_position,
+            "corr_method": corr_method,
+            "detail_scope": detail_scope,
+        }
+
     def _build_validation_summary(
         self,
         data: pd.DataFrame,