SuperModelingFactory 0.3.8__tar.gz → 0.3.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Explainability/Coalition_Structure.py +6 -4
- supermodelingfactory-0.3.10/Modeling_Tool/Feature/Feature_Screen.py +396 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Feature/Weighted_Screen.py +33 -37
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Feature/__init__.py +12 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Pipeline/__init__.py +3 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Pipeline/credit_model.py +16 -27
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Pipeline/feature_validation.py +168 -0
- supermodelingfactory-0.3.10/Modeling_Tool/Pipeline/screening_artifact.py +123 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/WOE_Plot_Tool.py +1 -2
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/__init__.py +14 -1
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/PKG-INFO +1 -1
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/SuperModelingFactory.egg-info/PKG-INFO +1 -1
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/SuperModelingFactory.egg-info/SOURCES.txt +2 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/pyproject.toml +1 -1
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/setup.py +1 -1
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/ExcelMaster/ExcelFormatTool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/ExcelMaster/ExcelMaster.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/ExcelMaster/Template.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/ExcelMaster/Utility.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/ExcelMaster/__init__.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/LICENSE +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/MANIFEST.in +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/Binning_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/Binning_Tool.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/Check_DuckDB_Compatibility.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/Json_Data_Converter.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/Model_Registry_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/ODPS_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/Parallel_Engine.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/Parallel_ODPS_Manager.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/Proc_Compare.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/Slope_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/Slope_Tool.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/XOR_Encryptor.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/XOR_Encryptor.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/__init__.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/kDataFrame.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/kDataFrame.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/sample_weight_utils.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Core/utils.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Eval/Evaluation_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Eval/Evaluation_Tool.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Eval/Model_Eval_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Eval/Model_Eval_Tool.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Eval/__init__.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Eval/evaluate_model.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Eval/evaluate_model.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Eval/weighted_eval_utils.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Explainability/Model_Explainer.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Explainability/__init__.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Feature/Distribution_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Feature/Distribution_Tool.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Feature/Feature_Insights.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Feature/Feature_Insights.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Feature/PSI_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Feature/PSI_Tool.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Model/Backward_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Model/Backward_Tool.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Model/GBM_Search_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Model/GBM_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Model/GBM_Tool.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Model/LRM_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Model/LRM_Tool.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Model/__init__.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Pipeline/_common.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Pipeline/mock_sample.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Pipeline/reject_inference.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Pipeline/sample_analysis.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Pipeline/score_comparison.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Pipeline/score_consistency_uat.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Sample/Distribution_Adaptation.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Sample/Distribution_Adaptation.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Sample/Reject_Infer.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Sample/Reject_Infer.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Sample/Sample_Split.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Sample/Sample_Split.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Sample/__init__.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/UAT/UAT_Consistency_Checker.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/UAT/__init__.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/WOE_Adapter.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/WOE_Adapter.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/WOE_Master.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/WOE_Master.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/WOE_Monotone_Binner.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/WOE_Monotone_Binner.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/WOE_Plot_Tool.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/WOE_Report_Builder.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/WOE_Report_Builder.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/WOE_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/WOE_Tool.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/__init__.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/plot_woe_tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/WOE/plot_woe_tool.pyi +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/ref_font/KaiTi.ttf +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/ref_font/WeiRuanYaHei.ttf +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/ref_font/__init__.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/ref_font/simsun.ttc +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/README.md +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Report/Report_Tool.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Report/__init__.py +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/SuperModelingFactory.egg-info/dependency_links.txt +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/SuperModelingFactory.egg-info/not-zip-safe +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/SuperModelingFactory.egg-info/requires.txt +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/SuperModelingFactory.egg-info/top_level.txt +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/requirements.txt +0 -0
- {supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/setup.cfg +0 -0
|
@@ -108,8 +108,9 @@ def _association_matrix(frame: pd.DataFrame, corr_method: str = "spearman") -> p
|
|
|
108
108
|
assoc = frame.corr(method=corr_method).abs()
|
|
109
109
|
assoc = assoc.reindex(index=frame.columns, columns=frame.columns).fillna(0.0)
|
|
110
110
|
assoc = assoc.clip(lower=0.0, upper=1.0)
|
|
111
|
-
|
|
112
|
-
|
|
111
|
+
arr = assoc.to_numpy(dtype=float, copy=True)
|
|
112
|
+
np.fill_diagonal(arr, 1.0)
|
|
113
|
+
return pd.DataFrame(arr, index=assoc.index, columns=assoc.columns)
|
|
113
114
|
|
|
114
115
|
|
|
115
116
|
def _as_numeric_frame(X: pd.DataFrame) -> pd.DataFrame:
|
|
@@ -136,8 +137,9 @@ def _correlation_distance(X: pd.DataFrame, corr_method: str = "spearman") -> pd.
|
|
|
136
137
|
frame = _as_numeric_frame(X)
|
|
137
138
|
assoc = _association_matrix(frame, corr_method=corr_method)
|
|
138
139
|
dist = 1.0 - assoc
|
|
139
|
-
|
|
140
|
-
|
|
140
|
+
arr = dist.to_numpy(dtype=float, copy=True)
|
|
141
|
+
np.fill_diagonal(arr, 0.0)
|
|
142
|
+
return pd.DataFrame(arr, index=dist.index, columns=dist.columns)
|
|
141
143
|
|
|
142
144
|
|
|
143
145
|
def compute_correlation_linkage(
|
|
@@ -0,0 +1,396 @@
|
|
|
1
|
+
"""Unified feature screening: PSI -> IV -> correlation dedup."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import warnings
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Mapping
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
|
|
13
|
+
from .Weighted_Screen import (
|
|
14
|
+
WeightedScreenResult,
|
|
15
|
+
_legacy_unweighted_screen,
|
|
16
|
+
_resolve_splits,
|
|
17
|
+
_summary_row,
|
|
18
|
+
_weighted_screen_impl,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
FeatureScreenResult = WeightedScreenResult
|
|
22
|
+
|
|
23
|
+
_MONOTONE_INIT_KEYS = frozenset({
|
|
24
|
+
"n_init_bins",
|
|
25
|
+
"min_bin_size",
|
|
26
|
+
"min_n_bins",
|
|
27
|
+
"eps",
|
|
28
|
+
"missing_woe",
|
|
29
|
+
"special_values",
|
|
30
|
+
"bin_label_decimals",
|
|
31
|
+
})
|
|
32
|
+
_MONOTONE_FIT_KEYS = frozenset({"chi2_binning", "chi2_p", "chi2_init_size", "n_jobs"})
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class FeatureScreenConfig:
|
|
37
|
+
psi_enabled: bool = True
|
|
38
|
+
psi_threshold: float = 0.2
|
|
39
|
+
psi_compare_splits: list[str] = field(default_factory=lambda: ["oos"])
|
|
40
|
+
psi_buckets: int = 10
|
|
41
|
+
psi_use_woe_bins: bool = False
|
|
42
|
+
iv_enabled: bool = True
|
|
43
|
+
iv_threshold: float = 0.02
|
|
44
|
+
iv_bins: int = 10
|
|
45
|
+
iv_min_bin_prop: float = 0.05
|
|
46
|
+
iv_equal_freq: bool = True
|
|
47
|
+
iv_use_woe_bins: bool = False
|
|
48
|
+
corr_enabled: bool = True
|
|
49
|
+
corr_threshold: float = 0.75
|
|
50
|
+
corr_max_iterations: int = 10
|
|
51
|
+
corr_use_woe_bins: bool = False
|
|
52
|
+
woe_engine: str = "equal_freq"
|
|
53
|
+
woe_fit_query: str | None = None
|
|
54
|
+
woe_params: dict[str, Any] = field(
|
|
55
|
+
default_factory=lambda: {"nbins": 10, "equal_freq": True, "min_bin_prop": 0.05}
|
|
56
|
+
)
|
|
57
|
+
monotone_woe_params: dict[str, Any] = field(
|
|
58
|
+
default_factory=lambda: {"n_init_bins": 20, "min_bin_size": 0.03, "min_n_bins": 2}
|
|
59
|
+
)
|
|
60
|
+
categorical_features: list[str] | None = None
|
|
61
|
+
plot_path: str | None = None
|
|
62
|
+
plot_outputs: bool = False
|
|
63
|
+
content: float = 1e-6
|
|
64
|
+
precision: int = 5
|
|
65
|
+
min_bin_prop: float = 0.05
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def screen_config_from_mapping(
|
|
69
|
+
mapping: Mapping[str, Any] | None,
|
|
70
|
+
*,
|
|
71
|
+
woe_engine: str | None = None,
|
|
72
|
+
woe_fit_query: str | None = None,
|
|
73
|
+
woe_params: Mapping[str, Any] | None = None,
|
|
74
|
+
monotone_woe_params: Mapping[str, Any] | None = None,
|
|
75
|
+
plot_path: str | None = None,
|
|
76
|
+
plot_outputs: bool = False,
|
|
77
|
+
) -> FeatureScreenConfig:
|
|
78
|
+
"""Build ``FeatureScreenConfig`` from a CM-style ``feature_selection`` dict."""
|
|
79
|
+
cfg = dict(mapping or {})
|
|
80
|
+
iv_nbins = int(cfg.get("iv_nbins", cfg.get("iv_bins", 10)))
|
|
81
|
+
return FeatureScreenConfig(
|
|
82
|
+
psi_enabled=bool(cfg.get("psi_enabled", True)),
|
|
83
|
+
psi_threshold=float(cfg.get("psi_threshold", 0.2)),
|
|
84
|
+
psi_compare_splits=list(cfg.get("psi_compare_splits", ["oos"])),
|
|
85
|
+
psi_buckets=int(cfg.get("psi_buckets", iv_nbins)),
|
|
86
|
+
psi_use_woe_bins=bool(cfg.get("psi_use_woe_bins", False)),
|
|
87
|
+
iv_enabled=bool(cfg.get("iv_enabled", True)),
|
|
88
|
+
iv_threshold=float(cfg.get("iv_threshold", 0.02)),
|
|
89
|
+
iv_bins=iv_nbins,
|
|
90
|
+
iv_min_bin_prop=float(cfg.get("iv_min_bin_prop", 0.05)),
|
|
91
|
+
iv_equal_freq=bool(cfg.get("iv_equal_freq", True)),
|
|
92
|
+
iv_use_woe_bins=bool(cfg.get("iv_use_woe_bins", False)),
|
|
93
|
+
corr_enabled=bool(cfg.get("corr_enabled", True)),
|
|
94
|
+
corr_threshold=float(cfg.get("corr_threshold", 0.75)),
|
|
95
|
+
corr_max_iterations=int(cfg.get("corr_max_iterations", 10)),
|
|
96
|
+
corr_use_woe_bins=bool(cfg.get("corr_use_woe_bins", False)),
|
|
97
|
+
woe_engine=str(woe_engine or cfg.get("woe_engine", "equal_freq")),
|
|
98
|
+
woe_fit_query=woe_fit_query if woe_fit_query is not None else cfg.get("woe_fit_query"),
|
|
99
|
+
woe_params=dict(FeatureScreenConfig().woe_params | dict(woe_params or cfg.get("woe_params") or {})),
|
|
100
|
+
monotone_woe_params=dict(
|
|
101
|
+
FeatureScreenConfig().monotone_woe_params | dict(monotone_woe_params or cfg.get("monotone_woe_params") or {})
|
|
102
|
+
),
|
|
103
|
+
categorical_features=list(cfg["categorical_features"]) if cfg.get("categorical_features") else None,
|
|
104
|
+
plot_path=plot_path,
|
|
105
|
+
plot_outputs=plot_outputs,
|
|
106
|
+
content=float(cfg.get("content", 1e-6)),
|
|
107
|
+
precision=int(cfg.get("precision", 5)),
|
|
108
|
+
min_bin_prop=float(cfg.get("min_bin_prop", cfg.get("iv_min_bin_prop", 0.05))),
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def fit_screening_woe_engine(
|
|
113
|
+
train: pd.DataFrame,
|
|
114
|
+
features: list[str],
|
|
115
|
+
target_col: str,
|
|
116
|
+
*,
|
|
117
|
+
woe_engine: str = "monotone",
|
|
118
|
+
woe_fit_query: str | None = None,
|
|
119
|
+
woe_params: Mapping[str, Any] | None = None,
|
|
120
|
+
monotone_woe_params: Mapping[str, Any] | None = None,
|
|
121
|
+
categorical_features: list[str] | None = None,
|
|
122
|
+
) -> Any:
|
|
123
|
+
"""Fit a WOE engine on INS for screening steps that reuse bin boundaries."""
|
|
124
|
+
from Modeling_Tool.Pipeline._common import apply_woe_fit_query
|
|
125
|
+
|
|
126
|
+
fit_ins, _ = apply_woe_fit_query(train, woe_fit_query, target=target_col)
|
|
127
|
+
if woe_engine.lower() == "monotone":
|
|
128
|
+
from Modeling_Tool import MonotoneWOEBinner
|
|
129
|
+
|
|
130
|
+
params = dict(monotone_woe_params or {})
|
|
131
|
+
init_params = {k: v for k, v in params.items() if k in _MONOTONE_INIT_KEYS}
|
|
132
|
+
fit_params = {k: v for k, v in params.items() if k in _MONOTONE_FIT_KEYS}
|
|
133
|
+
from Modeling_Tool.Pipeline._common import as_list
|
|
134
|
+
|
|
135
|
+
categorical = [col for col in as_list(categorical_features) if col in features]
|
|
136
|
+
numeric = [col for col in features if col not in set(categorical)]
|
|
137
|
+
binner = MonotoneWOEBinner(
|
|
138
|
+
feature_cols=numeric,
|
|
139
|
+
target_col=target_col,
|
|
140
|
+
cate_feats=categorical,
|
|
141
|
+
**init_params,
|
|
142
|
+
)
|
|
143
|
+
binner.fit(fit_ins, **fit_params)
|
|
144
|
+
return binner
|
|
145
|
+
|
|
146
|
+
from Modeling_Tool import WOE_Master
|
|
147
|
+
|
|
148
|
+
master_params = dict(woe_params or {})
|
|
149
|
+
woe_suffix = master_params.pop("woe_suffix", "_woe")
|
|
150
|
+
missing_ref_value = master_params.pop("missing_ref_value", -999999)
|
|
151
|
+
master = WOE_Master(
|
|
152
|
+
train_data=fit_ins,
|
|
153
|
+
varlist=features,
|
|
154
|
+
dep=target_col,
|
|
155
|
+
graph_save_dir=None,
|
|
156
|
+
woe_suffix=woe_suffix,
|
|
157
|
+
missing_ref_value=missing_ref_value,
|
|
158
|
+
)
|
|
159
|
+
master.fit(**master_params)
|
|
160
|
+
return master
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _needs_woe_bins(config: FeatureScreenConfig) -> bool:
|
|
164
|
+
return config.psi_use_woe_bins or config.iv_use_woe_bins or config.corr_use_woe_bins
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _resolve_screening_binner(
|
|
168
|
+
splits: dict[str, pd.DataFrame],
|
|
169
|
+
feature_cols: list[str],
|
|
170
|
+
target_col: str,
|
|
171
|
+
config: FeatureScreenConfig,
|
|
172
|
+
prefit_woe_engine: Any | None,
|
|
173
|
+
) -> Any | None:
|
|
174
|
+
if prefit_woe_engine is not None:
|
|
175
|
+
return prefit_woe_engine
|
|
176
|
+
if not _needs_woe_bins(config):
|
|
177
|
+
return None
|
|
178
|
+
return fit_screening_woe_engine(
|
|
179
|
+
splits["ins"],
|
|
180
|
+
feature_cols,
|
|
181
|
+
target_col,
|
|
182
|
+
woe_engine=config.woe_engine,
|
|
183
|
+
woe_fit_query=config.woe_fit_query,
|
|
184
|
+
woe_params=config.woe_params,
|
|
185
|
+
monotone_woe_params=config.monotone_woe_params,
|
|
186
|
+
categorical_features=config.categorical_features,
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _woe_bins_unweighted_screen(
|
|
191
|
+
splits: dict[str, pd.DataFrame],
|
|
192
|
+
feature_cols: list[str],
|
|
193
|
+
target_col: str,
|
|
194
|
+
config: FeatureScreenConfig,
|
|
195
|
+
*,
|
|
196
|
+
prefit_woe_engine: Any | None = None,
|
|
197
|
+
) -> FeatureScreenResult:
|
|
198
|
+
from Modeling_Tool import CorrelationFilter, PSICalculator, VarExtractionInsights
|
|
199
|
+
|
|
200
|
+
ins, oos, oot = splits["ins"], splits["oos"], splits["oot"]
|
|
201
|
+
current = list(feature_cols)
|
|
202
|
+
summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, None)]
|
|
203
|
+
binner = _resolve_screening_binner(splits, feature_cols, target_col, config, prefit_woe_engine)
|
|
204
|
+
|
|
205
|
+
psi_table = pd.DataFrame(columns=["var", "psi_ins_oos", "psi_ins_oot", "psi_max"])
|
|
206
|
+
if config.psi_enabled:
|
|
207
|
+
psi_calc = (
|
|
208
|
+
PSICalculator(buckets=config.psi_buckets, binning_engine=binner)
|
|
209
|
+
if config.psi_use_woe_bins and binner is not None
|
|
210
|
+
else PSICalculator(buckets=config.psi_buckets)
|
|
211
|
+
)
|
|
212
|
+
psi_frames = []
|
|
213
|
+
if "oos" in config.psi_compare_splits and len(oos) > 0:
|
|
214
|
+
psi_oos = psi_calc.calculate(expected_df=ins, current_data=oos, varlist=current)
|
|
215
|
+
psi_oos = psi_oos.rename(columns={"psi": "psi_ins_oos"})[["var", "psi_ins_oos"]]
|
|
216
|
+
psi_frames.append(psi_oos)
|
|
217
|
+
if "oot" in config.psi_compare_splits and len(oot) > 0:
|
|
218
|
+
psi_oot = psi_calc.calculate(expected_df=ins, current_data=oot, varlist=current)
|
|
219
|
+
psi_oot = psi_oot.rename(columns={"psi": "psi_ins_oot"})[["var", "psi_ins_oot"]]
|
|
220
|
+
psi_frames.append(psi_oot)
|
|
221
|
+
|
|
222
|
+
if psi_frames:
|
|
223
|
+
psi_table = psi_frames[0]
|
|
224
|
+
for frame in psi_frames[1:]:
|
|
225
|
+
psi_table = psi_table.merge(frame, on="var", how="outer")
|
|
226
|
+
for col in ("psi_ins_oos", "psi_ins_oot"):
|
|
227
|
+
if col not in psi_table.columns:
|
|
228
|
+
psi_table[col] = np.nan
|
|
229
|
+
compare_cols = [c for c in ("psi_ins_oos", "psi_ins_oot") if c in psi_table.columns]
|
|
230
|
+
psi_table["psi_max"] = psi_table[compare_cols].max(axis=1, skipna=True)
|
|
231
|
+
keep = psi_table.loc[psi_table["psi_max"] < config.psi_threshold, "var"].tolist()
|
|
232
|
+
n_before = len(current)
|
|
233
|
+
current = keep or current
|
|
234
|
+
summary_rows.append(_summary_row("psi", n_before, len(current), config.psi_threshold, None))
|
|
235
|
+
|
|
236
|
+
iv_table = pd.DataFrame(columns=["var", "iv_weighted", "n_bins", "missing_rate"])
|
|
237
|
+
if config.iv_enabled:
|
|
238
|
+
use_binner = config.iv_use_woe_bins and binner is not None
|
|
239
|
+
vi = VarExtractionInsights(
|
|
240
|
+
data=ins,
|
|
241
|
+
dep=target_col,
|
|
242
|
+
plot_path=config.plot_path or "",
|
|
243
|
+
nbins=config.iv_bins,
|
|
244
|
+
equal_freq=config.iv_equal_freq,
|
|
245
|
+
min_bin_prop=config.iv_min_bin_prop,
|
|
246
|
+
woe_binner=binner if use_binner else None,
|
|
247
|
+
woe_engine="monotone" if use_binner else "master",
|
|
248
|
+
)
|
|
249
|
+
iv = vi.get_var_analysis_report(data=ins, varlist=current, dep=target_col, iv_cut=0.0)
|
|
250
|
+
if config.plot_path and config.plot_outputs and current:
|
|
251
|
+
Path(config.plot_path, "overall").mkdir(parents=True, exist_ok=True)
|
|
252
|
+
plot_data = ins.copy()
|
|
253
|
+
plot_data["_smf_plot_group"] = "overall"
|
|
254
|
+
vi.plot_woe(
|
|
255
|
+
data=plot_data,
|
|
256
|
+
varlist=current,
|
|
257
|
+
plot_group="_smf_plot_group",
|
|
258
|
+
plot_dirname="overall",
|
|
259
|
+
plot_path=config.plot_path,
|
|
260
|
+
)
|
|
261
|
+
if iv is not None and not iv.empty:
|
|
262
|
+
iv_table = iv.rename(columns={"iv": "iv_weighted"})[["var", "iv_weighted", "n_bins", "missing_rate"]]
|
|
263
|
+
keep = iv.loc[iv["iv"] >= config.iv_threshold, "var"].tolist()
|
|
264
|
+
n_before = len(current)
|
|
265
|
+
current = keep or current
|
|
266
|
+
summary_rows.append(_summary_row("iv", n_before, len(current), config.iv_threshold, None))
|
|
267
|
+
|
|
268
|
+
corr_dropped = pd.DataFrame(columns=["var_a", "var_b", "corr", "iv_a", "iv_b", "kept", "dropped"])
|
|
269
|
+
if config.corr_enabled and len(current) > 1:
|
|
270
|
+
use_binner = config.corr_use_woe_bins and binner is not None
|
|
271
|
+
cf = CorrelationFilter(
|
|
272
|
+
data=ins[current + [target_col]],
|
|
273
|
+
dep=target_col,
|
|
274
|
+
corr_cutpoint=config.corr_threshold,
|
|
275
|
+
woe_binner=binner if use_binner else None,
|
|
276
|
+
woe_engine="monotone" if use_binner else "master",
|
|
277
|
+
)
|
|
278
|
+
n_before = len(current)
|
|
279
|
+
current = cf.remove_highly_correlated(current, max_iterations=config.corr_max_iterations)
|
|
280
|
+
summary_rows.append(_summary_row("corr", n_before, len(current), config.corr_threshold, None))
|
|
281
|
+
|
|
282
|
+
summary_rows.append(_summary_row("final", len(feature_cols), len(current), None, None))
|
|
283
|
+
return FeatureScreenResult(
|
|
284
|
+
selected_features=list(current),
|
|
285
|
+
iv_table=iv_table,
|
|
286
|
+
psi_table=psi_table,
|
|
287
|
+
corr_dropped=corr_dropped,
|
|
288
|
+
summary=pd.DataFrame(summary_rows),
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def feature_screen(
|
|
293
|
+
splits: dict[str, pd.DataFrame],
|
|
294
|
+
feature_cols: list[str],
|
|
295
|
+
target_col: str,
|
|
296
|
+
*,
|
|
297
|
+
weight_col: str | None = None,
|
|
298
|
+
config: FeatureScreenConfig | None = None,
|
|
299
|
+
prefit_woe_engine: Any | None = None,
|
|
300
|
+
) -> FeatureScreenResult:
|
|
301
|
+
"""Run PSI -> IV -> correlation screening on pre-split INS/OOS/OOT frames."""
|
|
302
|
+
cfg = config or FeatureScreenConfig()
|
|
303
|
+
use_woe_bins = _needs_woe_bins(cfg)
|
|
304
|
+
|
|
305
|
+
if weight_col is not None:
|
|
306
|
+
if use_woe_bins:
|
|
307
|
+
warnings.warn(
|
|
308
|
+
"Weighted feature screening with psi/iv/corr_use_woe_bins is not fully "
|
|
309
|
+
"supported yet; falling back to weighted equal-frequency bins.",
|
|
310
|
+
stacklevel=2,
|
|
311
|
+
)
|
|
312
|
+
return _weighted_screen_impl(
|
|
313
|
+
splits,
|
|
314
|
+
feature_cols,
|
|
315
|
+
target_col,
|
|
316
|
+
weight_col,
|
|
317
|
+
psi_enabled=cfg.psi_enabled,
|
|
318
|
+
psi_threshold=cfg.psi_threshold,
|
|
319
|
+
psi_compare_splits=list(cfg.psi_compare_splits),
|
|
320
|
+
iv_enabled=cfg.iv_enabled,
|
|
321
|
+
iv_threshold=cfg.iv_threshold,
|
|
322
|
+
iv_bins=cfg.iv_bins,
|
|
323
|
+
min_bin_prop=cfg.min_bin_prop,
|
|
324
|
+
corr_enabled=cfg.corr_enabled,
|
|
325
|
+
corr_threshold=cfg.corr_threshold,
|
|
326
|
+
corr_max_iterations=cfg.corr_max_iterations,
|
|
327
|
+
content=cfg.content,
|
|
328
|
+
precision=cfg.precision,
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
if use_woe_bins or prefit_woe_engine is not None:
|
|
332
|
+
return _woe_bins_unweighted_screen(
|
|
333
|
+
splits,
|
|
334
|
+
feature_cols,
|
|
335
|
+
target_col,
|
|
336
|
+
cfg,
|
|
337
|
+
prefit_woe_engine=prefit_woe_engine,
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
return _legacy_unweighted_screen(
|
|
341
|
+
splits,
|
|
342
|
+
feature_cols,
|
|
343
|
+
target_col,
|
|
344
|
+
psi_enabled=cfg.psi_enabled,
|
|
345
|
+
psi_threshold=cfg.psi_threshold,
|
|
346
|
+
psi_compare_splits=list(cfg.psi_compare_splits),
|
|
347
|
+
iv_enabled=cfg.iv_enabled,
|
|
348
|
+
iv_threshold=cfg.iv_threshold,
|
|
349
|
+
iv_bins=cfg.iv_bins,
|
|
350
|
+
iv_min_bin_prop=cfg.iv_min_bin_prop,
|
|
351
|
+
corr_enabled=cfg.corr_enabled,
|
|
352
|
+
corr_threshold=cfg.corr_threshold,
|
|
353
|
+
corr_max_iterations=cfg.corr_max_iterations,
|
|
354
|
+
psi_buckets=cfg.psi_buckets,
|
|
355
|
+
plot_path=cfg.plot_path,
|
|
356
|
+
plot_outputs=cfg.plot_outputs,
|
|
357
|
+
iv_equal_freq=cfg.iv_equal_freq,
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def feature_screen_from_dataframe(
|
|
362
|
+
data: pd.DataFrame,
|
|
363
|
+
feature_cols: list[str],
|
|
364
|
+
target_col: str,
|
|
365
|
+
split_col: str,
|
|
366
|
+
*,
|
|
367
|
+
weight_col: str | None = None,
|
|
368
|
+
config: FeatureScreenConfig | None = None,
|
|
369
|
+
prefit_woe_engine: Any | None = None,
|
|
370
|
+
) -> FeatureScreenResult:
|
|
371
|
+
"""Convenience wrapper that resolves INS/OOS/OOT from a tagged dataframe."""
|
|
372
|
+
exclude = {target_col, split_col}
|
|
373
|
+
if weight_col:
|
|
374
|
+
exclude.add(weight_col)
|
|
375
|
+
features = [c for c in feature_cols if c not in exclude and c in data.columns]
|
|
376
|
+
splits = _resolve_splits(data, split_col)
|
|
377
|
+
if weight_col is not None and weight_col not in data.columns:
|
|
378
|
+
raise KeyError(f"Missing weight_col {weight_col!r}")
|
|
379
|
+
return feature_screen(
|
|
380
|
+
splits,
|
|
381
|
+
features,
|
|
382
|
+
target_col,
|
|
383
|
+
weight_col=weight_col,
|
|
384
|
+
config=config,
|
|
385
|
+
prefit_woe_engine=prefit_woe_engine,
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
__all__ = [
|
|
390
|
+
"FeatureScreenConfig",
|
|
391
|
+
"FeatureScreenResult",
|
|
392
|
+
"feature_screen",
|
|
393
|
+
"feature_screen_from_dataframe",
|
|
394
|
+
"fit_screening_woe_engine",
|
|
395
|
+
"screen_config_from_mapping",
|
|
396
|
+
]
|
{supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Feature/Weighted_Screen.py
RENAMED
|
@@ -482,6 +482,14 @@ def weighted_feature_screen(
|
|
|
482
482
|
plot_path: str | None = None,
|
|
483
483
|
plot_outputs: bool = False,
|
|
484
484
|
iv_equal_freq: bool = True,
|
|
485
|
+
psi_use_woe_bins: bool = False,
|
|
486
|
+
iv_use_woe_bins: bool = False,
|
|
487
|
+
corr_use_woe_bins: bool = False,
|
|
488
|
+
woe_engine: str = "equal_freq",
|
|
489
|
+
woe_fit_query: str | None = None,
|
|
490
|
+
woe_params: dict[str, Any] | None = None,
|
|
491
|
+
monotone_woe_params: dict[str, Any] | None = None,
|
|
492
|
+
prefit_woe_engine: Any | None = None,
|
|
485
493
|
) -> WeightedScreenResult:
|
|
486
494
|
"""Run PSI -> IV -> correlation screening with optional sample weights.
|
|
487
495
|
|
|
@@ -490,6 +498,8 @@ def weighted_feature_screen(
|
|
|
490
498
|
``CreditModelPipeline`` (IV uses tree binning via ``VarExtractionInsights``).
|
|
491
499
|
Weighted screening uses equal-frequency bins on weighted quantiles.
|
|
492
500
|
"""
|
|
501
|
+
from .Feature_Screen import FeatureScreenConfig, feature_screen_from_dataframe
|
|
502
|
+
|
|
493
503
|
if iv_min_bin_prop is None:
|
|
494
504
|
iv_min_bin_prop = min_bin_prop
|
|
495
505
|
if psi_buckets is None:
|
|
@@ -497,52 +507,38 @@ def weighted_feature_screen(
|
|
|
497
507
|
if psi_compare_splits is None:
|
|
498
508
|
psi_compare_splits = ["oos", "oot"]
|
|
499
509
|
|
|
500
|
-
|
|
501
|
-
if weight_col:
|
|
502
|
-
exclude.add(weight_col)
|
|
503
|
-
features = [c for c in feature_cols if c not in exclude and c in data.columns]
|
|
504
|
-
|
|
505
|
-
splits = _resolve_splits(data, split_col)
|
|
506
|
-
|
|
507
|
-
if weight_col is None:
|
|
508
|
-
return _legacy_unweighted_screen(
|
|
509
|
-
splits,
|
|
510
|
-
features,
|
|
511
|
-
target_col,
|
|
512
|
-
psi_enabled=psi_enabled,
|
|
513
|
-
psi_threshold=psi_threshold,
|
|
514
|
-
psi_compare_splits=list(psi_compare_splits),
|
|
515
|
-
iv_enabled=iv_enabled,
|
|
516
|
-
iv_threshold=iv_threshold,
|
|
517
|
-
iv_bins=iv_bins,
|
|
518
|
-
iv_min_bin_prop=iv_min_bin_prop,
|
|
519
|
-
corr_enabled=corr_enabled,
|
|
520
|
-
corr_threshold=corr_threshold,
|
|
521
|
-
corr_max_iterations=corr_max_iterations,
|
|
522
|
-
psi_buckets=psi_buckets,
|
|
523
|
-
plot_path=plot_path,
|
|
524
|
-
plot_outputs=plot_outputs,
|
|
525
|
-
iv_equal_freq=iv_equal_freq,
|
|
526
|
-
)
|
|
527
|
-
|
|
528
|
-
if weight_col not in data.columns:
|
|
529
|
-
raise KeyError(f"Missing weight_col {weight_col!r}")
|
|
530
|
-
|
|
531
|
-
return _weighted_screen_impl(
|
|
532
|
-
splits,
|
|
533
|
-
features,
|
|
534
|
-
target_col,
|
|
535
|
-
weight_col,
|
|
510
|
+
config = FeatureScreenConfig(
|
|
536
511
|
psi_enabled=psi_enabled,
|
|
537
512
|
psi_threshold=psi_threshold,
|
|
538
513
|
psi_compare_splits=list(psi_compare_splits),
|
|
514
|
+
psi_buckets=psi_buckets,
|
|
515
|
+
psi_use_woe_bins=psi_use_woe_bins,
|
|
539
516
|
iv_enabled=iv_enabled,
|
|
540
517
|
iv_threshold=iv_threshold,
|
|
541
518
|
iv_bins=iv_bins,
|
|
542
|
-
|
|
519
|
+
iv_min_bin_prop=iv_min_bin_prop,
|
|
520
|
+
iv_equal_freq=iv_equal_freq,
|
|
521
|
+
iv_use_woe_bins=iv_use_woe_bins,
|
|
543
522
|
corr_enabled=corr_enabled,
|
|
544
523
|
corr_threshold=corr_threshold,
|
|
545
524
|
corr_max_iterations=corr_max_iterations,
|
|
525
|
+
corr_use_woe_bins=corr_use_woe_bins,
|
|
526
|
+
woe_engine=woe_engine,
|
|
527
|
+
woe_fit_query=woe_fit_query,
|
|
528
|
+
woe_params=dict(woe_params or {}),
|
|
529
|
+
monotone_woe_params=dict(monotone_woe_params or {}),
|
|
530
|
+
plot_path=plot_path,
|
|
531
|
+
plot_outputs=plot_outputs,
|
|
546
532
|
content=content,
|
|
547
533
|
precision=precision,
|
|
534
|
+
min_bin_prop=min_bin_prop,
|
|
535
|
+
)
|
|
536
|
+
return feature_screen_from_dataframe(
|
|
537
|
+
data,
|
|
538
|
+
feature_cols,
|
|
539
|
+
target_col,
|
|
540
|
+
split_col,
|
|
541
|
+
weight_col=weight_col,
|
|
542
|
+
config=config,
|
|
543
|
+
prefit_woe_engine=prefit_woe_engine,
|
|
548
544
|
)
|
{supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Feature/__init__.py
RENAMED
|
@@ -25,6 +25,14 @@ from .WOE_Engine_Feature_Patch import (
|
|
|
25
25
|
CorrelationFilter,
|
|
26
26
|
)
|
|
27
27
|
|
|
28
|
+
from .Feature_Screen import (
|
|
29
|
+
FeatureScreenConfig,
|
|
30
|
+
FeatureScreenResult,
|
|
31
|
+
feature_screen,
|
|
32
|
+
feature_screen_from_dataframe,
|
|
33
|
+
fit_screening_woe_engine,
|
|
34
|
+
screen_config_from_mapping,
|
|
35
|
+
)
|
|
28
36
|
from .Weighted_Screen import (
|
|
29
37
|
WeightedScreenResult,
|
|
30
38
|
weighted_feature_screen,
|
|
@@ -43,6 +51,10 @@ __all__ = [
|
|
|
43
51
|
'PSICalculator', 'calculate_psi', 'calculate_within_psi',
|
|
44
52
|
'calculate_psi_within_dataset', 'calculate_multivar_psi_two_sets',
|
|
45
53
|
|
|
54
|
+
# Feature_Screen
|
|
55
|
+
'FeatureScreenConfig', 'FeatureScreenResult', 'feature_screen',
|
|
56
|
+
'feature_screen_from_dataframe', 'fit_screening_woe_engine', 'screen_config_from_mapping',
|
|
57
|
+
|
|
46
58
|
# Weighted_Screen
|
|
47
59
|
'WeightedScreenResult', 'weighted_feature_screen',
|
|
48
60
|
]
|
{supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Pipeline/__init__.py
RENAMED
|
@@ -29,6 +29,7 @@ from .feature_validation import (
|
|
|
29
29
|
FeatureValidationPipelineConfig,
|
|
30
30
|
FeatureValidationPipelineResult,
|
|
31
31
|
)
|
|
32
|
+
from .screening_artifact import FeatureScreeningArtifact, screen_result_to_summary
|
|
32
33
|
|
|
33
34
|
__all__ = [
|
|
34
35
|
"RejectInferencePipeline",
|
|
@@ -52,4 +53,6 @@ __all__ = [
|
|
|
52
53
|
"FeatureValidationPipeline",
|
|
53
54
|
"FeatureValidationPipelineConfig",
|
|
54
55
|
"FeatureValidationPipelineResult",
|
|
56
|
+
"FeatureScreeningArtifact",
|
|
57
|
+
"screen_result_to_summary",
|
|
55
58
|
]
|
{supermodelingfactory-0.3.8 → supermodelingfactory-0.3.10}/Modeling_Tool/Pipeline/credit_model.py
RENAMED
|
@@ -429,17 +429,10 @@ class CreditModelPipeline:
|
|
|
429
429
|
splits: dict[str, pd.DataFrame],
|
|
430
430
|
feature_cols: list[str],
|
|
431
431
|
) -> tuple[dict[str, Any], list[str]]:
|
|
432
|
-
from Modeling_Tool.Feature.
|
|
432
|
+
from Modeling_Tool.Feature.Feature_Screen import feature_screen, screen_config_from_mapping
|
|
433
433
|
|
|
434
434
|
cfg = self.config
|
|
435
435
|
fs_cfg = cfg.feature_selection
|
|
436
|
-
split_col = "_smf_split"
|
|
437
|
-
screen_frames = []
|
|
438
|
-
for name, df in splits.items():
|
|
439
|
-
part = df.copy()
|
|
440
|
-
part[split_col] = name
|
|
441
|
-
screen_frames.append(part)
|
|
442
|
-
screen_data = pd.concat(screen_frames, ignore_index=True)
|
|
443
436
|
|
|
444
437
|
plot_path = None
|
|
445
438
|
if fs_cfg.get("iv_enabled", True):
|
|
@@ -447,28 +440,24 @@ class CreditModelPipeline:
|
|
|
447
440
|
if cfg.write_outputs and cfg.plot_outputs:
|
|
448
441
|
make_dirs(plot_path, Path(plot_path) / "overall")
|
|
449
442
|
|
|
443
|
+
screen_config = screen_config_from_mapping(
|
|
444
|
+
fs_cfg,
|
|
445
|
+
woe_engine=cfg.woe_engine,
|
|
446
|
+
woe_fit_query=cfg.woe_fit_query,
|
|
447
|
+
woe_params=cfg.woe_params,
|
|
448
|
+
monotone_woe_params=cfg.monotone_woe_params,
|
|
449
|
+
plot_path=plot_path,
|
|
450
|
+
plot_outputs=bool(cfg.write_outputs and cfg.plot_outputs),
|
|
451
|
+
)
|
|
452
|
+
|
|
450
453
|
summary: dict[str, Any] = {"initial_features": list(feature_cols)}
|
|
451
454
|
try:
|
|
452
|
-
result =
|
|
453
|
-
|
|
454
|
-
feature_cols
|
|
455
|
-
|
|
456
|
-
split_col=split_col,
|
|
455
|
+
result = feature_screen(
|
|
456
|
+
splits,
|
|
457
|
+
feature_cols,
|
|
458
|
+
cfg.target_col,
|
|
457
459
|
weight_col=cfg.weight_col,
|
|
458
|
-
|
|
459
|
-
psi_threshold=float(fs_cfg.get("psi_threshold", 0.2)),
|
|
460
|
-
psi_compare_splits=list(fs_cfg.get("psi_compare_splits", ["oos"])),
|
|
461
|
-
iv_enabled=bool(fs_cfg.get("iv_enabled", True)),
|
|
462
|
-
iv_threshold=float(fs_cfg.get("iv_threshold", 0.02)),
|
|
463
|
-
corr_enabled=bool(fs_cfg.get("corr_enabled", True)),
|
|
464
|
-
corr_threshold=float(fs_cfg.get("corr_threshold", 0.75)),
|
|
465
|
-
iv_bins=int(fs_cfg.get("iv_nbins", 10)),
|
|
466
|
-
iv_min_bin_prop=float(fs_cfg.get("iv_min_bin_prop", 0.05)),
|
|
467
|
-
corr_max_iterations=int(fs_cfg.get("corr_max_iterations", 10)),
|
|
468
|
-
psi_buckets=int(fs_cfg.get("psi_buckets", fs_cfg.get("iv_nbins", 10))),
|
|
469
|
-
plot_path=plot_path,
|
|
470
|
-
plot_outputs=bool(cfg.write_outputs and cfg.plot_outputs),
|
|
471
|
-
iv_equal_freq=bool(fs_cfg.get("iv_equal_freq", True)),
|
|
460
|
+
config=screen_config,
|
|
472
461
|
)
|
|
473
462
|
summary = self._screen_result_to_summary(result, feature_cols)
|
|
474
463
|
return summary, list(result.selected_features)
|