SuperModelingFactory 0.3.12__tar.gz → 0.3.14__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Feature/Feature_Screen.py +37 -8
  2. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Feature/Weighted_Screen.py +202 -17
  3. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Pipeline/__init__.py +2 -0
  4. supermodelingfactory-0.3.14/Modeling_Tool/Pipeline/orchestrator.py +50 -0
  5. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/__init__.py +2 -2
  6. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/PKG-INFO +1 -1
  7. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/SuperModelingFactory.egg-info/PKG-INFO +1 -1
  8. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/SuperModelingFactory.egg-info/SOURCES.txt +1 -0
  9. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/pyproject.toml +1 -1
  10. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/setup.py +1 -1
  11. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/ExcelMaster/ExcelFormatTool.py +0 -0
  12. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/ExcelMaster/ExcelMaster.py +0 -0
  13. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/ExcelMaster/Template.py +0 -0
  14. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/ExcelMaster/Utility.py +0 -0
  15. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/ExcelMaster/__init__.py +0 -0
  16. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/LICENSE +0 -0
  17. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/MANIFEST.in +0 -0
  18. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/Binning_Tool.py +0 -0
  19. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/Binning_Tool.pyi +0 -0
  20. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/Check_DuckDB_Compatibility.py +0 -0
  21. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/Json_Data_Converter.py +0 -0
  22. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/Model_Registry_Tool.py +0 -0
  23. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/ODPS_Tool.py +0 -0
  24. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/Parallel_Engine.py +0 -0
  25. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/Parallel_ODPS_Manager.py +0 -0
  26. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/Proc_Compare.py +0 -0
  27. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/Slope_Tool.py +0 -0
  28. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/Slope_Tool.pyi +0 -0
  29. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/XOR_Encryptor.py +0 -0
  30. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/XOR_Encryptor.pyi +0 -0
  31. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/__init__.py +0 -0
  32. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/kDataFrame.py +0 -0
  33. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/kDataFrame.pyi +0 -0
  34. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/sample_weight_utils.py +0 -0
  35. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Core/utils.py +0 -0
  36. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Eval/Evaluation_Tool.py +0 -0
  37. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Eval/Evaluation_Tool.pyi +0 -0
  38. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Eval/Model_Eval_Tool.py +0 -0
  39. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Eval/Model_Eval_Tool.pyi +0 -0
  40. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Eval/__init__.py +0 -0
  41. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Eval/evaluate_model.py +0 -0
  42. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Eval/evaluate_model.pyi +0 -0
  43. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Eval/weighted_eval_utils.py +0 -0
  44. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Explainability/Coalition_Structure.py +0 -0
  45. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Explainability/Model_Explainer.py +0 -0
  46. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Explainability/__init__.py +0 -0
  47. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Feature/Distribution_Tool.py +0 -0
  48. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Feature/Distribution_Tool.pyi +0 -0
  49. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Feature/Feature_Insights.py +0 -0
  50. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Feature/Feature_Insights.pyi +0 -0
  51. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Feature/PSI_Tool.py +0 -0
  52. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Feature/PSI_Tool.pyi +0 -0
  53. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py +0 -0
  54. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Feature/__init__.py +0 -0
  55. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Model/Backward_Tool.py +0 -0
  56. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Model/Backward_Tool.pyi +0 -0
  57. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Model/GBM_Search_Tool.py +0 -0
  58. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Model/GBM_Tool.py +0 -0
  59. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Model/GBM_Tool.pyi +0 -0
  60. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Model/LRM_Tool.py +0 -0
  61. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Model/LRM_Tool.pyi +0 -0
  62. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Model/__init__.py +0 -0
  63. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Pipeline/_common.py +0 -0
  64. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Pipeline/credit_model.py +0 -0
  65. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Pipeline/feature_validation.py +0 -0
  66. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Pipeline/mock_sample.py +0 -0
  67. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Pipeline/reject_inference.py +0 -0
  68. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Pipeline/sample_analysis.py +0 -0
  69. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Pipeline/score_comparison.py +0 -0
  70. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Pipeline/score_consistency_uat.py +0 -0
  71. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Pipeline/screening_artifact.py +0 -0
  72. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Sample/Distribution_Adaptation.py +0 -0
  73. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Sample/Distribution_Adaptation.pyi +0 -0
  74. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Sample/Reject_Infer.py +0 -0
  75. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Sample/Reject_Infer.pyi +0 -0
  76. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Sample/Sample_Split.py +0 -0
  77. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Sample/Sample_Split.pyi +0 -0
  78. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/Sample/__init__.py +0 -0
  79. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/UAT/UAT_Consistency_Checker.py +0 -0
  80. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/UAT/__init__.py +0 -0
  81. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/WOE_Adapter.py +0 -0
  82. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/WOE_Adapter.pyi +0 -0
  83. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/WOE_Master.py +0 -0
  84. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/WOE_Master.pyi +0 -0
  85. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/WOE_Monotone_Binner.py +0 -0
  86. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/WOE_Monotone_Binner.pyi +0 -0
  87. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/WOE_Plot_Tool.py +0 -0
  88. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/WOE_Plot_Tool.pyi +0 -0
  89. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/WOE_Report_Builder.py +0 -0
  90. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/WOE_Report_Builder.pyi +0 -0
  91. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/WOE_Tool.py +0 -0
  92. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/WOE_Tool.pyi +0 -0
  93. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/__init__.py +0 -0
  94. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/plot_woe_tool.py +0 -0
  95. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/WOE/plot_woe_tool.pyi +0 -0
  96. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/ref_font/KaiTi.ttf +0 -0
  97. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/ref_font/WeiRuanYaHei.ttf +0 -0
  98. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/ref_font/__init__.py +0 -0
  99. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Modeling_Tool/ref_font/simsun.ttc +0 -0
  100. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/README.md +0 -0
  101. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Report/Report_Tool.py +0 -0
  102. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/Report/__init__.py +0 -0
  103. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/SuperModelingFactory.egg-info/dependency_links.txt +0 -0
  104. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/SuperModelingFactory.egg-info/not-zip-safe +0 -0
  105. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/SuperModelingFactory.egg-info/requires.txt +0 -0
  106. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/SuperModelingFactory.egg-info/top_level.txt +0 -0
  107. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/requirements.txt +0 -0
  108. {supermodelingfactory-0.3.12 → supermodelingfactory-0.3.14}/setup.cfg +0 -0
@@ -4,7 +4,7 @@ from __future__ import annotations
4
4
 
5
5
  from dataclasses import dataclass, field
6
6
  from pathlib import Path
7
- from typing import Any, Mapping
7
+ from typing import Any, Literal, Mapping
8
8
 
9
9
  import numpy as np
10
10
  import pandas as pd
@@ -13,14 +13,15 @@ from Modeling_Tool.Core.sample_weight_utils import resolve_sample_weight
13
13
 
14
14
  from .Weighted_Screen import (
15
15
  WeightedScreenResult,
16
+ _apply_stage_keep,
16
17
  _corr_dedup_weighted,
17
18
  _legacy_unweighted_screen,
18
19
  _psi_from_distributions,
19
20
  _resolve_splits,
20
21
  _summary_row,
21
22
  _weighted_bin_distribution,
23
+ _weighted_corr_for_screen,
22
24
  _weighted_iv_from_assigned_bins,
23
- _weighted_pearson_corr_matrix,
24
25
  _weighted_screen_impl,
25
26
  )
26
27
 
@@ -55,6 +56,8 @@ class FeatureScreenConfig:
55
56
  corr_threshold: float = 0.75
56
57
  corr_max_iterations: int = 10
57
58
  corr_use_woe_bins: bool = False
59
+ corr_nan_policy: Literal["pairwise", "median_fill", "raise"] = "pairwise"
60
+ on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn"
58
61
  woe_engine: str = "equal_freq"
59
62
  woe_fit_query: str | None = None
60
63
  woe_params: dict[str, Any] = field(
@@ -100,6 +103,8 @@ def screen_config_from_mapping(
100
103
  corr_threshold=float(cfg.get("corr_threshold", 0.75)),
101
104
  corr_max_iterations=int(cfg.get("corr_max_iterations", 10)),
102
105
  corr_use_woe_bins=bool(cfg.get("corr_use_woe_bins", False)),
106
+ corr_nan_policy=str(cfg.get("corr_nan_policy", "pairwise")), # type: ignore[arg-type]
107
+ on_empty_stage=str(cfg.get("on_empty_stage", "keep_all_warn")), # type: ignore[arg-type]
103
108
  woe_engine=str(woe_engine or cfg.get("woe_engine", "equal_freq")),
104
109
  woe_fit_query=woe_fit_query if woe_fit_query is not None else cfg.get("woe_fit_query"),
105
110
  woe_params=dict(FeatureScreenConfig().woe_params | dict(woe_params or cfg.get("woe_params") or {})),
@@ -236,7 +241,10 @@ def _woe_bins_unweighted_screen(
236
241
  psi_table["psi_max"] = psi_table[compare_cols].max(axis=1, skipna=True)
237
242
  keep = psi_table.loc[psi_table["psi_max"] < config.psi_threshold, "var"].tolist()
238
243
  n_before = len(current)
239
- current = keep or current
244
+ current = _apply_stage_keep(
245
+ current, keep, "psi", summary_rows,
246
+ on_empty_stage=config.on_empty_stage, threshold=config.psi_threshold,
247
+ )
240
248
  summary_rows.append(_summary_row("psi", n_before, len(current), config.psi_threshold, None))
241
249
 
242
250
  iv_table = pd.DataFrame(columns=["var", "iv_weighted", "n_bins", "missing_rate"])
@@ -268,7 +276,10 @@ def _woe_bins_unweighted_screen(
268
276
  iv_table = iv.rename(columns={"iv": "iv_weighted"})[["var", "iv_weighted", "n_bins", "missing_rate"]]
269
277
  keep = iv.loc[iv["iv"] >= config.iv_threshold, "var"].tolist()
270
278
  n_before = len(current)
271
- current = keep or current
279
+ current = _apply_stage_keep(
280
+ current, keep, "iv", summary_rows,
281
+ on_empty_stage=config.on_empty_stage, threshold=config.iv_threshold,
282
+ )
272
283
  summary_rows.append(_summary_row("iv", n_before, len(current), config.iv_threshold, None))
273
284
 
274
285
  corr_dropped = pd.DataFrame(columns=["var_a", "var_b", "corr", "iv_a", "iv_b", "kept", "dropped"])
@@ -360,7 +371,11 @@ def _weighted_woe_bins_screen(
360
371
  psi_table["psi_max"] = psi_table[compare_cols].max(axis=1, skipna=True)
361
372
  keep = psi_table.loc[psi_table["psi_max"] < config.psi_threshold, "var"].tolist()
362
373
  n_before = len(current)
363
- current = [v for v in current if v in keep] or current
374
+ current = _apply_stage_keep(
375
+ current, keep, "psi", summary_rows,
376
+ on_empty_stage=config.on_empty_stage, weight_col=weight_col,
377
+ threshold=config.psi_threshold, intersect=True,
378
+ )
364
379
  summary_rows.append(_summary_row("psi", n_before, len(current), config.psi_threshold, weight_col))
365
380
  else:
366
381
  psi_table["psi_max"] = pd.Series(dtype=float)
@@ -387,15 +402,24 @@ def _weighted_woe_bins_screen(
387
402
  if not iv_table.empty:
388
403
  keep = iv_table.loc[iv_table["iv_weighted"] >= config.iv_threshold, "var"].tolist()
389
404
  n_before = len(current)
390
- current = [v for v in current if v in keep] or current
405
+ current = _apply_stage_keep(
406
+ current, keep, "iv", summary_rows,
407
+ on_empty_stage=config.on_empty_stage, weight_col=weight_col,
408
+ threshold=config.iv_threshold, intersect=True,
409
+ )
391
410
  summary_rows.append(_summary_row("iv", n_before, len(current), config.iv_threshold, weight_col))
392
411
  else:
393
412
  iv_table = pd.DataFrame(columns=["var", "iv_weighted", "n_bins", "missing_rate"])
394
413
 
395
414
  corr_dropped = pd.DataFrame(columns=["var_a", "var_b", "corr", "iv_a", "iv_b", "kept", "dropped"])
396
415
  if config.corr_enabled and len(current) > 1:
397
- X = ins[current].astype(float).to_numpy()
398
- corr = _weighted_pearson_corr_matrix(X, w_ins)
416
+ corr = _weighted_corr_for_screen(
417
+ ins, current, w_ins,
418
+ corr_use_woe_bins=config.corr_use_woe_bins,
419
+ corr_nan_policy=config.corr_nan_policy,
420
+ adapter=adapter,
421
+ binner=binner,
422
+ )
399
423
  iv_map = dict(zip(iv_table["var"], iv_table["iv_weighted"])) if not iv_table.empty else {}
400
424
  n_before = len(current)
401
425
  current, corr_dropped = _corr_dedup_weighted(
@@ -457,6 +481,10 @@ def feature_screen(
457
481
  corr_max_iterations=cfg.corr_max_iterations,
458
482
  content=cfg.content,
459
483
  precision=cfg.precision,
484
+ corr_use_woe_bins=cfg.corr_use_woe_bins,
485
+ corr_nan_policy=cfg.corr_nan_policy,
486
+ on_empty_stage=cfg.on_empty_stage,
487
+ prefit_woe_engine=prefit_woe_engine,
460
488
  )
461
489
 
462
490
  if use_woe_bins or prefit_woe_engine is not None:
@@ -486,6 +514,7 @@ def feature_screen(
486
514
  plot_path=cfg.plot_path,
487
515
  plot_outputs=cfg.plot_outputs,
488
516
  iv_equal_freq=cfg.iv_equal_freq,
517
+ on_empty_stage=cfg.on_empty_stage,
489
518
  )
490
519
 
491
520
 
@@ -2,9 +2,10 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ import warnings
5
6
  from dataclasses import dataclass
6
7
  from pathlib import Path
7
- from typing import Any
8
+ from typing import Any, Literal
8
9
 
9
10
  import numpy as np
10
11
  import pandas as pd
@@ -185,21 +186,178 @@ def _weighted_iv_for_var(
185
186
  return iv, len(rows), missing_rate
186
187
 
187
188
 
188
- def _weighted_pearson_corr_matrix(X: np.ndarray, w: np.ndarray) -> np.ndarray:
189
+ def _weighted_median(x: np.ndarray, w: np.ndarray) -> float:
190
+ mask = np.isfinite(x)
191
+ if not mask.any():
192
+ return np.nan
193
+ xv = x[mask].astype(float)
194
+ wv = w[mask].astype(float)
195
+ order = np.argsort(xv)
196
+ xv = xv[order]
197
+ wv = wv[order]
198
+ cumw = np.cumsum(wv)
199
+ total = float(cumw[-1])
200
+ if total <= 0:
201
+ return float(np.median(xv))
202
+ idx = min(int(np.searchsorted(cumw, total / 2.0, side="right")), len(xv) - 1)
203
+ return float(xv[idx])
204
+
205
+
206
+ def _weighted_corr_pair(xi: np.ndarray, xj: np.ndarray, w: np.ndarray) -> float:
207
+ mask = np.isfinite(xi) & np.isfinite(xj)
208
+ if mask.sum() < 2:
209
+ return 0.0
210
+ wi = w[mask].astype(float)
211
+ xi = xi[mask].astype(float)
212
+ xj = xj[mask].astype(float)
213
+ w_sum = wi.sum()
214
+ if w_sum <= 0:
215
+ c = np.corrcoef(xi, xj)
216
+ return float(c[0, 1]) if np.isfinite(c[0, 1]) else 0.0
217
+ w_norm = wi / w_sum
218
+ mi = np.average(xi, weights=w_norm)
219
+ mj = np.average(xj, weights=w_norm)
220
+ di = xi - mi
221
+ dj = xj - mj
222
+ cov = np.average(di * dj, weights=w_norm)
223
+ si = np.sqrt(np.average(di * di, weights=w_norm))
224
+ sj = np.sqrt(np.average(dj * dj, weights=w_norm))
225
+ if si == 0 or sj == 0:
226
+ return 0.0
227
+ return float(cov / (si * sj))
228
+
229
+
230
+ def _weighted_pearson_corr_matrix(
231
+ X: np.ndarray,
232
+ w: np.ndarray,
233
+ *,
234
+ nan_policy: Literal["pairwise", "median_fill", "raise"] = "pairwise",
235
+ ) -> np.ndarray:
189
236
  if X.shape[1] == 0:
190
237
  return np.empty((0, 0))
191
238
  w = np.asarray(w, dtype=float)
239
+ k = X.shape[1]
240
+
241
+ if nan_policy == "raise" and np.any(~np.isfinite(X)):
242
+ raise ValueError(
243
+ "weighted correlation input contains NaN; set corr_nan_policy to "
244
+ "'pairwise' or 'median_fill' to proceed"
245
+ )
246
+
247
+ nan_mask = ~np.isfinite(X)
248
+ nan_fraction = float(nan_mask.sum()) / float(X.size) if X.size else 0.0
249
+
250
+ if nan_policy == "pairwise":
251
+ corr = np.eye(k, dtype=float)
252
+ insufficient_pairs = 0
253
+ total_pairs = k * (k - 1) // 2
254
+ for i in range(k):
255
+ for j in range(i + 1, k):
256
+ c = _weighted_corr_pair(X[:, i], X[:, j], w)
257
+ corr[i, j] = c
258
+ corr[j, i] = c
259
+ overlap = int((np.isfinite(X[:, i]) & np.isfinite(X[:, j])).sum())
260
+ if overlap < 2:
261
+ insufficient_pairs += 1
262
+ if total_pairs > 0 and insufficient_pairs / total_pairs > 0.01:
263
+ warnings.warn(
264
+ f"[feature_screen] weighted corr: >1% of feature pairs have insufficient "
265
+ f"overlapping observations ({insufficient_pairs}/{total_pairs})",
266
+ stacklevel=3,
267
+ )
268
+ return corr
269
+
270
+ X_use = X.astype(float).copy()
271
+ if nan_policy == "median_fill":
272
+ for j in range(k):
273
+ col = X_use[:, j]
274
+ bad = ~np.isfinite(col)
275
+ if bad.any():
276
+ fill = _weighted_median(col, w)
277
+ if np.isfinite(fill):
278
+ col[bad] = fill
279
+ X_use[:, j] = col
280
+
192
281
  w_sum = w.sum()
193
282
  if w_sum <= 0:
194
- return np.corrcoef(X, rowvar=False)
195
- w_norm = w / w_sum
196
- mean = np.average(X, axis=0, weights=w_norm)
197
- Xc = X - mean
198
- cov = np.cov(Xc, rowvar=False, aweights=w, ddof=0)
199
- std = np.sqrt(np.diag(cov))
200
- std[std == 0] = np.nan
201
- corr = cov / np.outer(std, std)
202
- return np.nan_to_num(corr, nan=0.0)
283
+ out = np.corrcoef(X_use, rowvar=False)
284
+ out = np.nan_to_num(out, nan=0.0)
285
+ else:
286
+ w_norm = w / w_sum
287
+ mean = np.average(X_use, axis=0, weights=w_norm)
288
+ Xc = X_use - mean
289
+ cov = np.cov(Xc, rowvar=False, aweights=w, ddof=0)
290
+ std = np.sqrt(np.diag(cov))
291
+ std[std == 0] = np.nan
292
+ out = cov / np.outer(std, std)
293
+ out = np.nan_to_num(out, nan=0.0)
294
+
295
+ if nan_fraction > 0.01:
296
+ warnings.warn(
297
+ f"[feature_screen] weighted corr: {nan_fraction:.1%} of matrix entries were NaN "
298
+ f"before computation (policy={nan_policy!r})",
299
+ stacklevel=3,
300
+ )
301
+ return out
302
+
303
+
304
+ def _weighted_corr_for_screen(
305
+ ins: pd.DataFrame,
306
+ current: list[str],
307
+ w_ins: np.ndarray,
308
+ *,
309
+ corr_use_woe_bins: bool,
310
+ corr_nan_policy: Literal["pairwise", "median_fill", "raise"],
311
+ adapter: Any | None = None,
312
+ binner: Any | None = None,
313
+ ) -> np.ndarray:
314
+ """Build weighted correlation matrix for screening (WOE or raw-value path)."""
315
+ from Modeling_Tool.WOE.WOE_Adapter import as_woe_engine
316
+
317
+ if corr_use_woe_bins:
318
+ eng = adapter if adapter is not None else (as_woe_engine(binner) if binner is not None else None)
319
+ if eng is not None:
320
+ suffix = getattr(eng, "woe_suffix", "_woe")
321
+ woe_ins = eng.transform(ins, varlist=current)
322
+ cols = [f"{v}{suffix}" for v in current if f"{v}{suffix}" in woe_ins.columns]
323
+ if cols:
324
+ X = woe_ins[cols].to_numpy(dtype=float)
325
+ return _weighted_pearson_corr_matrix(X, w_ins, nan_policy="pairwise")
326
+ X = ins[current].astype(float).to_numpy()
327
+ return _weighted_pearson_corr_matrix(X, w_ins, nan_policy=corr_nan_policy)
328
+
329
+
330
+ def _apply_stage_keep(
331
+ current: list[str],
332
+ keep: list[str],
333
+ stage: str,
334
+ summary_rows: list[dict],
335
+ *,
336
+ on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn",
337
+ weight_col: str | None = None,
338
+ threshold: Any = None,
339
+ intersect: bool = False,
340
+ ) -> list[str]:
341
+ if intersect:
342
+ keep_set = set(keep)
343
+ new_current = [v for v in current if v in keep_set]
344
+ else:
345
+ new_current = list(keep) if keep else []
346
+ if new_current:
347
+ return new_current
348
+ if on_empty_stage == "raise":
349
+ raise ValueError(
350
+ f"feature screen stage {stage!r} eliminated all {len(current)} variables; "
351
+ f"on_empty_stage='raise'"
352
+ )
353
+ warnings.warn(
354
+ f"[feature_screen] stage {stage!r} 全军覆没, 按 keep_all_warn 保留全部 {len(current)} 个",
355
+ stacklevel=3,
356
+ )
357
+ summary_rows.append(
358
+ _summary_row(f"{stage}_fallback", len(current), len(current), threshold, weight_col)
359
+ )
360
+ return current
203
361
 
204
362
 
205
363
  def _high_corr_pairs(varlist: list[str], corr: np.ndarray, threshold: float) -> pd.DataFrame:
@@ -280,6 +438,7 @@ def _legacy_unweighted_screen(
280
438
  plot_path: str | None,
281
439
  plot_outputs: bool,
282
440
  iv_equal_freq: bool = True,
441
+ on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn",
283
442
  ) -> WeightedScreenResult:
284
443
  from Modeling_Tool import CorrelationFilter, PSICalculator, VarExtractionInsights
285
444
 
@@ -314,7 +473,10 @@ def _legacy_unweighted_screen(
314
473
  psi_table["psi_max"] = psi_table[compare_cols].max(axis=1, skipna=True)
315
474
  keep = psi_table.loc[psi_table["psi_max"] < psi_threshold, "var"].tolist()
316
475
  n_before = len(current)
317
- current = keep or current
476
+ current = _apply_stage_keep(
477
+ current, keep, "psi", summary_rows,
478
+ on_empty_stage=on_empty_stage, threshold=psi_threshold,
479
+ )
318
480
  summary_rows.append(_summary_row("psi", n_before, len(current), psi_threshold, None))
319
481
 
320
482
  iv_table = pd.DataFrame(columns=["var", "iv_weighted", "n_bins", "missing_rate"])
@@ -342,7 +504,10 @@ def _legacy_unweighted_screen(
342
504
  iv_table = iv.rename(columns={"iv": "iv_weighted"})[["var", "iv_weighted", "n_bins", "missing_rate"]]
343
505
  keep = iv.loc[iv["iv"] >= iv_threshold, "var"].tolist()
344
506
  n_before = len(current)
345
- current = keep or current
507
+ current = _apply_stage_keep(
508
+ current, keep, "iv", summary_rows,
509
+ on_empty_stage=on_empty_stage, threshold=iv_threshold,
510
+ )
346
511
  summary_rows.append(_summary_row("iv", n_before, len(current), iv_threshold, None))
347
512
 
348
513
  corr_dropped = pd.DataFrame(columns=["var_a", "var_b", "corr", "iv_a", "iv_b", "kept", "dropped"])
@@ -384,6 +549,10 @@ def _weighted_screen_impl(
384
549
  corr_max_iterations: int,
385
550
  content: float,
386
551
  precision: int,
552
+ corr_use_woe_bins: bool = False,
553
+ corr_nan_policy: Literal["pairwise", "median_fill", "raise"] = "pairwise",
554
+ on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn",
555
+ prefit_woe_engine: Any | None = None,
387
556
  ) -> WeightedScreenResult:
388
557
  ins = splits["ins"]
389
558
  oos = splits["oos"]
@@ -432,7 +601,11 @@ def _weighted_screen_impl(
432
601
  psi_table["psi_max"] = psi_table[compare_cols].max(axis=1, skipna=True)
433
602
  keep = psi_table.loc[psi_table["psi_max"] < psi_threshold, "var"].tolist()
434
603
  n_before = len(current)
435
- current = [v for v in current if v in keep] or current
604
+ current = _apply_stage_keep(
605
+ current, keep, "psi", summary_rows,
606
+ on_empty_stage=on_empty_stage, weight_col=weight_col,
607
+ threshold=psi_threshold, intersect=True,
608
+ )
436
609
  summary_rows.append(_summary_row("psi", n_before, len(current), psi_threshold, weight_col))
437
610
  else:
438
611
  psi_table["psi_max"] = pd.Series(dtype=float)
@@ -464,15 +637,23 @@ def _weighted_screen_impl(
464
637
  if not iv_table.empty:
465
638
  keep = iv_table.loc[iv_table["iv_weighted"] >= iv_threshold, "var"].tolist()
466
639
  n_before = len(current)
467
- current = [v for v in current if v in keep] or current
640
+ current = _apply_stage_keep(
641
+ current, keep, "iv", summary_rows,
642
+ on_empty_stage=on_empty_stage, weight_col=weight_col,
643
+ threshold=iv_threshold, intersect=True,
644
+ )
468
645
  summary_rows.append(_summary_row("iv", n_before, len(current), iv_threshold, weight_col))
469
646
  else:
470
647
  iv_table = pd.DataFrame(columns=["var", "iv_weighted", "n_bins", "missing_rate"])
471
648
 
472
649
  corr_dropped = pd.DataFrame(columns=["var_a", "var_b", "corr", "iv_a", "iv_b", "kept", "dropped"])
473
650
  if corr_enabled and len(current) > 1:
474
- X = ins[current].astype(float).to_numpy()
475
- corr = _weighted_pearson_corr_matrix(X, w_ins)
651
+ corr = _weighted_corr_for_screen(
652
+ ins, current, w_ins,
653
+ corr_use_woe_bins=corr_use_woe_bins,
654
+ corr_nan_policy=corr_nan_policy,
655
+ binner=prefit_woe_engine,
656
+ )
476
657
  iv_map = dict(zip(iv_table["var"], iv_table["iv_weighted"])) if not iv_table.empty else {}
477
658
  n_before = len(current)
478
659
  current, corr_dropped = _corr_dedup_weighted(
@@ -521,6 +702,8 @@ def weighted_feature_screen(
521
702
  woe_params: dict[str, Any] | None = None,
522
703
  monotone_woe_params: dict[str, Any] | None = None,
523
704
  prefit_woe_engine: Any | None = None,
705
+ corr_nan_policy: Literal["pairwise", "median_fill", "raise"] = "pairwise",
706
+ on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn",
524
707
  ) -> WeightedScreenResult:
525
708
  """Run PSI -> IV -> correlation screening with optional sample weights.
526
709
 
@@ -563,6 +746,8 @@ def weighted_feature_screen(
563
746
  content=content,
564
747
  precision=precision,
565
748
  min_bin_prop=min_bin_prop,
749
+ corr_nan_policy=corr_nan_policy,
750
+ on_empty_stage=on_empty_stage,
566
751
  )
567
752
  return feature_screen_from_dataframe(
568
753
  data,
@@ -29,6 +29,7 @@ from .feature_validation import (
29
29
  FeatureValidationPipelineConfig,
30
30
  FeatureValidationPipelineResult,
31
31
  )
32
+ from .orchestrator import run_modeling_from_validation
32
33
  from .screening_artifact import FeatureScreeningArtifact, screen_result_to_summary
33
34
 
34
35
  __all__ = [
@@ -55,4 +56,5 @@ __all__ = [
55
56
  "FeatureValidationPipelineResult",
56
57
  "FeatureScreeningArtifact",
57
58
  "screen_result_to_summary",
59
+ "run_modeling_from_validation",
58
60
  ]
@@ -0,0 +1,50 @@
1
+ """High-level orchestration helpers across SMF pipelines."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import replace
6
+ from typing import Any
7
+
8
+ import pandas as pd
9
+
10
+ from .credit_model import CreditModelPipeline, CreditModelPipelineConfig, CreditModelPipelineResult
11
+ from .feature_validation import (
12
+ FeatureValidationPipeline,
13
+ FeatureValidationPipelineConfig,
14
+ FeatureValidationPipelineResult,
15
+ )
16
+ from .screening_artifact import FeatureScreeningArtifact
17
+
18
+
19
+ def run_modeling_from_validation(
20
+ data: pd.DataFrame,
21
+ *,
22
+ fvp_config: FeatureValidationPipelineConfig | None = None,
23
+ cm_config: CreditModelPipelineConfig | None = None,
24
+ selection_enabled: bool = True,
25
+ reuse_screening_woe: bool = True,
26
+ ) -> tuple[FeatureValidationPipelineResult, CreditModelPipelineResult]:
27
+ """Run feature validation/selection, then credit modeling on the same dataset."""
28
+ fvp_cfg = fvp_config or FeatureValidationPipelineConfig()
29
+ if selection_enabled and not fvp_cfg.selection_enabled:
30
+ fvp_cfg = replace(fvp_cfg, selection_enabled=True)
31
+
32
+ fvp_result = FeatureValidationPipeline(fvp_cfg).run(data)
33
+ artifact = FeatureScreeningArtifact.from_fvp_result(fvp_result)
34
+
35
+ cm_cfg = cm_config or CreditModelPipelineConfig()
36
+ cm_overrides: dict[str, Any] = {
37
+ "screening_artifact": artifact,
38
+ "reuse_screening_woe": reuse_screening_woe,
39
+ "target_col": artifact.target_col,
40
+ "weight_col": artifact.weight_col,
41
+ }
42
+ if artifact.selected_features:
43
+ cm_overrides["feature_cols"] = list(artifact.selected_features)
44
+ cm_cfg = replace(cm_cfg, **cm_overrides)
45
+
46
+ cm_result = CreditModelPipeline(cm_cfg).run(data)
47
+ return fvp_result, cm_result
48
+
49
+
50
+ __all__ = ["run_modeling_from_validation"]
@@ -1,7 +1,7 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  __author__ = "Jingkai Sun"
4
- __version__ = "0.3.12"
4
+ __version__ = "0.3.14"
5
5
 
6
6
  # Core module - base data structures
7
7
  from .Core import (
@@ -116,7 +116,7 @@ _PIPELINE_EXPORTS = frozenset({
116
116
  'SampleAnalysisPipeline', 'SampleAnalysisPipelineConfig', 'SampleAnalysisPipelineResult',
117
117
  'MockSamplePipeline', 'MockSamplePipelineConfig', 'MockSamplePipelineResult',
118
118
  'FeatureValidationPipeline', 'FeatureValidationPipelineConfig', 'FeatureValidationPipelineResult',
119
- 'FeatureScreeningArtifact', 'screen_result_to_summary',
119
+ 'FeatureScreeningArtifact', 'screen_result_to_summary', 'run_modeling_from_validation',
120
120
  })
121
121
 
122
122
  def __getattr__(name):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SuperModelingFactory
3
- Version: 0.3.12
3
+ Version: 0.3.14
4
4
  Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
5
5
  Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
6
6
  Author: Kyle Sun
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SuperModelingFactory
3
- Version: 0.3.12
3
+ Version: 0.3.14
4
4
  Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
5
5
  Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
6
6
  Author: Kyle Sun
@@ -62,6 +62,7 @@ Modeling_Tool/Pipeline/_common.py
62
62
  Modeling_Tool/Pipeline/credit_model.py
63
63
  Modeling_Tool/Pipeline/feature_validation.py
64
64
  Modeling_Tool/Pipeline/mock_sample.py
65
+ Modeling_Tool/Pipeline/orchestrator.py
65
66
  Modeling_Tool/Pipeline/reject_inference.py
66
67
  Modeling_Tool/Pipeline/sample_analysis.py
67
68
  Modeling_Tool/Pipeline/score_comparison.py
@@ -12,7 +12,7 @@ build-backend = "setuptools.build_meta"
12
12
 
13
13
  [project]
14
14
  name = "SuperModelingFactory"
15
- version = "0.3.12"
15
+ version = "0.3.14"
16
16
  description = "Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting."
17
17
  readme = "README.md"
18
18
  requires-python = ">=3.10"
@@ -15,7 +15,7 @@ def _read(path: str) -> str:
15
15
 
16
16
  setup(
17
17
  name="SuperModelingFactory",
18
- version=os.environ.get("SMF_VERSION", "0.3.12"),
18
+ version=os.environ.get("SMF_VERSION", "0.3.14"),
19
19
  description="Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.",
20
20
  long_description=_read("README.md"),
21
21
  long_description_content_type="text/markdown",