SuperModelingFactory 0.5.0__tar.gz → 0.5.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/ODPS_Tool.py +28 -5
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/Feature_Insights.py +24 -5
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/Feature_Screen.py +6 -5
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/PSI_Tool.py +126 -38
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/PSI_Tool.pyi +7 -7
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py +101 -14
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/Weighted_Screen.py +80 -42
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Pipeline/reject_inference.py +31 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/WOE_Master.py +39 -1
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/WOE_Master.pyi +1 -1
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/__init__.py +4 -1
- supermodelingfactory-0.5.2/Modeling_Tool/_utils/__init__.py +6 -0
- supermodelingfactory-0.5.2/Modeling_Tool/_utils/robust.py +55 -0
- supermodelingfactory-0.5.2/Modeling_Tool/_utils/sentinels.py +10 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/PKG-INFO +2 -2
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/README.md +1 -1
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/SuperModelingFactory.egg-info/PKG-INFO +2 -2
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/SuperModelingFactory.egg-info/SOURCES.txt +3 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/pyproject.toml +1 -1
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/setup.py +1 -1
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/ExcelMaster/ExcelFormatTool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/ExcelMaster/ExcelMaster.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/ExcelMaster/Template.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/ExcelMaster/Utility.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/ExcelMaster/__init__.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/LICENSE +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/MANIFEST.in +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/Binning_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/Binning_Tool.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/Check_DuckDB_Compatibility.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/Json_Data_Converter.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/Model_Registry_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/Parallel_Engine.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/Parallel_ODPS_Manager.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/Proc_Compare.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/Slope_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/Slope_Tool.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/XOR_Encryptor.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/XOR_Encryptor.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/__init__.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/kDataFrame.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/kDataFrame.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/sample_weight_utils.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Core/utils.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Eval/Evaluation_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Eval/Evaluation_Tool.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Eval/Model_Eval_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Eval/Model_Eval_Tool.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Eval/__init__.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Eval/evaluate_model.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Eval/evaluate_model.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Eval/weighted_eval_utils.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Explainability/Coalition_Structure.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Explainability/Model_Explainer.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Explainability/__init__.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/Distribution_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/Distribution_Tool.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/Feature_Insights.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/__init__.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Model/Backward_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Model/Backward_Tool.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Model/GBM_Search_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Model/GBM_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Model/GBM_Tool.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Model/LRM_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Model/LRM_Tool.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Model/__init__.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Pipeline/__init__.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Pipeline/_common.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Pipeline/credit_model.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Pipeline/feature_validation.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Pipeline/mock_sample.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Pipeline/orchestrator.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Pipeline/sample_analysis.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Pipeline/score_comparison.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Pipeline/score_consistency_uat.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Pipeline/screening_artifact.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Sample/Distribution_Adaptation.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Sample/Distribution_Adaptation.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Sample/Reject_Infer.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Sample/Reject_Infer.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Sample/Sample_Split.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Sample/Sample_Split.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Sample/__init__.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/UAT/UAT_Consistency_Checker.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/UAT/__init__.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/WOE_Adapter.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/WOE_Adapter.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/WOE_Monotone_Binner.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/WOE_Monotone_Binner.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/WOE_Plot_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/WOE_Plot_Tool.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/WOE_Report_Builder.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/WOE_Report_Builder.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/WOE_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/WOE_Tool.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/__init__.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/plot_woe_tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/WOE/plot_woe_tool.pyi +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/ref_font/KaiTi.ttf +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/ref_font/WeiRuanYaHei.ttf +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/ref_font/__init__.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/ref_font/simsun.ttc +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Report/Report_Tool.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Report/__init__.py +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/SuperModelingFactory.egg-info/dependency_links.txt +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/SuperModelingFactory.egg-info/not-zip-safe +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/SuperModelingFactory.egg-info/requires.txt +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/SuperModelingFactory.egg-info/top_level.txt +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/requirements.txt +0 -0
- {supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/setup.cfg +0 -0
|
@@ -17,6 +17,28 @@ pd.options.mode.chained_assignment = None # default='warn'
|
|
|
17
17
|
|
|
18
18
|
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
19
19
|
|
|
20
|
+
|
|
21
|
+
def _split_odps_table_name(table_name):
|
|
22
|
+
"""Return (qualifier, bare table name) for ODPS table identifiers.
|
|
23
|
+
|
|
24
|
+
MaxCompute accepts a project-qualified source in ``ALTER TABLE`` but the
|
|
25
|
+
``RENAME TO`` target must be the bare table name within the same project.
|
|
26
|
+
"""
|
|
27
|
+
table_name = str(table_name)
|
|
28
|
+
if "." not in table_name:
|
|
29
|
+
return "", table_name
|
|
30
|
+
qualifier, bare_name = table_name.rsplit(".", 1)
|
|
31
|
+
return qualifier, bare_name
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _make_related_odps_table_name(table_name, suffix):
|
|
35
|
+
qualifier, bare_name = _split_odps_table_name(table_name)
|
|
36
|
+
related_bare = f"{bare_name}{suffix}"
|
|
37
|
+
if qualifier:
|
|
38
|
+
return f"{qualifier}.{related_bare}", related_bare
|
|
39
|
+
return related_bare, related_bare
|
|
40
|
+
|
|
41
|
+
|
|
20
42
|
class ODPSRunner(object):
|
|
21
43
|
_wide_schema_patch_lock = threading.RLock()
|
|
22
44
|
_wide_schema_patch_ref_count = 0
|
|
@@ -292,8 +314,9 @@ class ODPSRunner(object):
|
|
|
292
314
|
# Atomic swap path (default from 0.4.2). If any step before the final
|
|
293
315
|
# swap fails, the original table is untouched.
|
|
294
316
|
ts = datetime.now().strftime('%Y%m%d%H%M%S%f')
|
|
295
|
-
|
|
296
|
-
|
|
317
|
+
_, target_rename_name = _split_odps_table_name(table_name)
|
|
318
|
+
tmp_name, _ = _make_related_odps_table_name(table_name, f"__tmp_{ts}")
|
|
319
|
+
old_name, old_rename_name = _make_related_odps_table_name(table_name, f"__old_{ts}")
|
|
297
320
|
|
|
298
321
|
# Step 1: write data into the temp table. If this fails, drop the
|
|
299
322
|
# partial temp table and re-raise; the target table is untouched.
|
|
@@ -321,7 +344,7 @@ class ODPSRunner(object):
|
|
|
321
344
|
target_exists = self.o.exist_table(table_name)
|
|
322
345
|
if target_exists:
|
|
323
346
|
try:
|
|
324
|
-
self.o.run_sql(f"ALTER TABLE {table_name} RENAME TO {
|
|
347
|
+
self.o.run_sql(f"ALTER TABLE {table_name} RENAME TO {old_rename_name};")
|
|
325
348
|
except Exception:
|
|
326
349
|
# Rename of the live target failed — do not touch it. Drop the
|
|
327
350
|
# tmp table so we don't leak it, then re-raise.
|
|
@@ -334,13 +357,13 @@ class ODPSRunner(object):
|
|
|
334
357
|
raise
|
|
335
358
|
|
|
336
359
|
try:
|
|
337
|
-
self.o.run_sql(f"ALTER TABLE {tmp_name} RENAME TO {
|
|
360
|
+
self.o.run_sql(f"ALTER TABLE {tmp_name} RENAME TO {target_rename_name};")
|
|
338
361
|
except Exception:
|
|
339
362
|
# tmp -> target rename failed. If we already moved the original,
|
|
340
363
|
# try to restore it so the caller is not left with a missing table.
|
|
341
364
|
if target_exists:
|
|
342
365
|
try:
|
|
343
|
-
self.o.run_sql(f"ALTER TABLE {old_name} RENAME TO {
|
|
366
|
+
self.o.run_sql(f"ALTER TABLE {old_name} RENAME TO {target_rename_name};")
|
|
344
367
|
logger.warning(
|
|
345
368
|
f"upload_df: swap of {tmp_name} -> {table_name} failed; restored original from {old_name}"
|
|
346
369
|
)
|
{supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/Feature_Insights.py
RENAMED
|
@@ -4,9 +4,11 @@
|
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
6
|
import pandas as pd
|
|
7
|
+
import warnings
|
|
7
8
|
from tqdm import tqdm
|
|
8
9
|
|
|
9
10
|
from .Distribution_Tool import proc_means_by_grp, proc_means_for_screening
|
|
11
|
+
from Modeling_Tool._utils.robust import smf_logger
|
|
10
12
|
import logging
|
|
11
13
|
logger = logging.getLogger(__name__)
|
|
12
14
|
|
|
@@ -108,6 +110,7 @@ class VarExtractionInsights:
|
|
|
108
110
|
self.seed = seed
|
|
109
111
|
self.missing_rate_ref = missing_rate_ref
|
|
110
112
|
self.spec_values = spec_values if spec_values is not None else []
|
|
113
|
+
self.failed_variables = []
|
|
111
114
|
|
|
112
115
|
@staticmethod
|
|
113
116
|
def remove_folder(file_path):
|
|
@@ -170,6 +173,7 @@ class VarExtractionInsights:
|
|
|
170
173
|
"""
|
|
171
174
|
if dep is None:
|
|
172
175
|
dep = self.dep
|
|
176
|
+
self.failed_variables = []
|
|
173
177
|
|
|
174
178
|
from Modeling_Tool.Eval.Model_Eval_Tool import get_gains_table
|
|
175
179
|
|
|
@@ -200,9 +204,27 @@ class VarExtractionInsights:
|
|
|
200
204
|
attr_iv['var'] = var
|
|
201
205
|
iv_info_res.append(attr_iv)
|
|
202
206
|
|
|
203
|
-
except TypeError:
|
|
207
|
+
except (TypeError, ValueError, KeyError, ZeroDivisionError) as exc:
|
|
208
|
+
row = smf_logger.record_and_continue(var, exc, stage="feature_insights")
|
|
209
|
+
self.failed_variables.append((row["feature"], row["exception_type"]))
|
|
204
210
|
continue
|
|
205
211
|
|
|
212
|
+
if self.failed_variables:
|
|
213
|
+
failed = [name for name, _ in self.failed_variables]
|
|
214
|
+
warnings.warn(
|
|
215
|
+
f"{len(failed)}/{len(varlist)} variables failed insight computation: "
|
|
216
|
+
f"{failed[:10]}{'...' if len(failed) > 10 else ''} — see smf_logger for details",
|
|
217
|
+
UserWarning,
|
|
218
|
+
stacklevel=2,
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
output_cols = [
|
|
222
|
+
'var', 'n_all', 'n', 'ks_in_gains', 'lift_in_gains', 'iv',
|
|
223
|
+
'n_bump', 'missing_rate', 'min', 'mean', 'max', 'n_bins'
|
|
224
|
+
]
|
|
225
|
+
if not iv_info_res:
|
|
226
|
+
return pd.DataFrame(columns=output_cols)
|
|
227
|
+
|
|
206
228
|
iv_info_res = pd.concat(iv_info_res).sort_values("IV", ascending=False)
|
|
207
229
|
|
|
208
230
|
high_iv_summary = iv_info_res.query(f"IV >= {iv_cut}").round(4)
|
|
@@ -222,10 +244,7 @@ class VarExtractionInsights:
|
|
|
222
244
|
how='left'
|
|
223
245
|
)
|
|
224
246
|
fnl_summary.columns = [x.lower() for x in fnl_summary.columns]
|
|
225
|
-
fnl_summary = fnl_summary[
|
|
226
|
-
'var', 'n_all', 'n', 'ks_in_gains', 'lift_in_gains', 'iv',
|
|
227
|
-
'n_bump', 'missing_rate', 'min', 'mean', 'max', 'n_bins'
|
|
228
|
-
]]
|
|
247
|
+
fnl_summary = fnl_summary[output_cols]
|
|
229
248
|
|
|
230
249
|
return fnl_summary
|
|
231
250
|
|
{supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/Feature_Screen.py
RENAMED
|
@@ -10,6 +10,7 @@ import numpy as np
|
|
|
10
10
|
import pandas as pd
|
|
11
11
|
|
|
12
12
|
from Modeling_Tool.Core.sample_weight_utils import resolve_sample_weight
|
|
13
|
+
from Modeling_Tool._utils.sentinels import SMF_MISSING_BIN
|
|
13
14
|
|
|
14
15
|
from .Weighted_Screen import (
|
|
15
16
|
WeightedScreenResult,
|
|
@@ -163,7 +164,7 @@ def fit_screening_woe_engine(
|
|
|
163
164
|
|
|
164
165
|
master_params = dict(woe_params or {})
|
|
165
166
|
woe_suffix = master_params.pop("woe_suffix", "_woe")
|
|
166
|
-
missing_ref_value = master_params.pop("missing_ref_value",
|
|
167
|
+
missing_ref_value = master_params.pop("missing_ref_value", SMF_MISSING_BIN)
|
|
167
168
|
master = WOE_Master(
|
|
168
169
|
train_data=fit_ins,
|
|
169
170
|
varlist=features,
|
|
@@ -224,7 +225,7 @@ def _woe_bins_unweighted_screen(
|
|
|
224
225
|
missing_rate_ref=config.missing_rate_ref,
|
|
225
226
|
on_empty_stage=config.on_empty_stage,
|
|
226
227
|
)
|
|
227
|
-
binner = _resolve_screening_binner(splits,
|
|
228
|
+
binner = _resolve_screening_binner(splits, current, target_col, config, prefit_woe_engine)
|
|
228
229
|
|
|
229
230
|
psi_table = pd.DataFrame(columns=["var", "psi_ins_oos", "psi_ins_oot", "psi_max"])
|
|
230
231
|
if config.psi_enabled:
|
|
@@ -357,7 +358,7 @@ def _weighted_woe_bins_screen(
|
|
|
357
358
|
if binner is None:
|
|
358
359
|
binner = fit_screening_woe_engine(
|
|
359
360
|
ins,
|
|
360
|
-
|
|
361
|
+
current,
|
|
361
362
|
target_col,
|
|
362
363
|
woe_engine=config.woe_engine,
|
|
363
364
|
woe_fit_query=config.woe_fit_query,
|
|
@@ -369,7 +370,7 @@ def _weighted_woe_bins_screen(
|
|
|
369
370
|
|
|
370
371
|
psi_records: list[dict] = []
|
|
371
372
|
if config.psi_enabled:
|
|
372
|
-
for var in
|
|
373
|
+
for var in current:
|
|
373
374
|
if var not in ins.columns or ins[var].nunique(dropna=False) <= 1:
|
|
374
375
|
continue
|
|
375
376
|
bins_ins = _bins_to_numpy(adapter.assign_bins(ins, var))
|
|
@@ -583,4 +584,4 @@ __all__ = [
|
|
|
583
584
|
"feature_screen_from_dataframe",
|
|
584
585
|
"fit_screening_woe_engine",
|
|
585
586
|
"screen_config_from_mapping",
|
|
586
|
-
]
|
|
587
|
+
]
|
|
@@ -20,6 +20,58 @@ from Modeling_Tool.Core.Binning_Tool import quick_binning
|
|
|
20
20
|
# ``Feature.Weighted_Screen`` and ``WOE_Adapter`` so downstream aggregators can
|
|
21
21
|
# recognise it without special-casing.
|
|
22
22
|
_MISSING_BIN = "__MISSING__"
|
|
23
|
+
_PSI_BUCKET_POLICIES = {"floor_1e6", "smooth_laplace", "exclude"}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _validate_psi_bucket_policy(policy: str, caller: str) -> None:
|
|
27
|
+
if policy not in _PSI_BUCKET_POLICIES:
|
|
28
|
+
raise ValueError(
|
|
29
|
+
f"{caller}: psi_missing_bucket_policy must be one of "
|
|
30
|
+
f"{sorted(_PSI_BUCKET_POLICIES)}; got {policy!r}."
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _psi_distributions_from_counts(
|
|
35
|
+
expected_count: pd.Series,
|
|
36
|
+
actual_count: pd.Series,
|
|
37
|
+
expected_total: float,
|
|
38
|
+
actual_total: float,
|
|
39
|
+
*,
|
|
40
|
+
content: float,
|
|
41
|
+
policy: str,
|
|
42
|
+
) -> tuple[pd.Series, pd.Series, pd.Series, pd.Series]:
|
|
43
|
+
_validate_psi_bucket_policy(policy, "_psi_distributions_from_counts")
|
|
44
|
+
all_bins = expected_count.index.union(actual_count.index)
|
|
45
|
+
expected_aligned = expected_count.reindex(all_bins, fill_value=0).astype(float)
|
|
46
|
+
actual_aligned = actual_count.reindex(all_bins, fill_value=0).astype(float)
|
|
47
|
+
one_sided = (expected_aligned == 0) | (actual_aligned == 0)
|
|
48
|
+
|
|
49
|
+
expected_total = float(expected_total) if expected_total > 0 else 1.0
|
|
50
|
+
actual_total = float(actual_total) if actual_total > 0 else 1.0
|
|
51
|
+
|
|
52
|
+
if policy == "exclude":
|
|
53
|
+
keep = ~one_sided
|
|
54
|
+
expected_aligned = expected_aligned[keep]
|
|
55
|
+
actual_aligned = actual_aligned[keep]
|
|
56
|
+
if expected_aligned.empty:
|
|
57
|
+
empty = pd.Series(dtype=float)
|
|
58
|
+
return empty, empty, empty, empty
|
|
59
|
+
expected_pct = expected_aligned / expected_total
|
|
60
|
+
actual_pct = actual_aligned / actual_total
|
|
61
|
+
elif policy == "smooth_laplace" and bool(one_sided.any()):
|
|
62
|
+
n_buckets = max(len(all_bins), 1)
|
|
63
|
+
expected_pct = (expected_aligned + 1.0) / (expected_total + n_buckets)
|
|
64
|
+
actual_pct = (actual_aligned + 1.0) / (actual_total + n_buckets)
|
|
65
|
+
else:
|
|
66
|
+
expected_pct = (expected_aligned / expected_total).clip(lower=content)
|
|
67
|
+
actual_pct = (actual_aligned / actual_total).clip(lower=content)
|
|
68
|
+
|
|
69
|
+
psi_values = (actual_pct - expected_pct) * np.log(actual_pct / expected_pct)
|
|
70
|
+
|
|
71
|
+
floor_expected = (expected_aligned / expected_total).clip(lower=content)
|
|
72
|
+
floor_actual = (actual_aligned / actual_total).clip(lower=content)
|
|
73
|
+
floor_values = (floor_actual - floor_expected) * np.log(floor_actual / floor_expected)
|
|
74
|
+
return expected_pct, actual_pct, psi_values, floor_values
|
|
23
75
|
|
|
24
76
|
# ============================================================================
|
|
25
77
|
# Classes
|
|
@@ -59,7 +111,8 @@ class PSICalculator:
|
|
|
59
111
|
min_bin_prop: float = 0.05,
|
|
60
112
|
content: float = 1e-6,
|
|
61
113
|
precision: int = 5,
|
|
62
|
-
missing_policy: str = "include"
|
|
114
|
+
missing_policy: str = "include",
|
|
115
|
+
psi_missing_bucket_policy: str = "smooth_laplace",
|
|
63
116
|
):
|
|
64
117
|
"""
|
|
65
118
|
Initialize PSICalculator with configuration parameters.
|
|
@@ -90,12 +143,14 @@ class PSICalculator:
|
|
|
90
143
|
f"PSICalculator.__init__: missing_policy must be one of "
|
|
91
144
|
f"'include', 'drop', 'warn_and_drop'; got {missing_policy!r}."
|
|
92
145
|
)
|
|
146
|
+
_validate_psi_bucket_policy(psi_missing_bucket_policy, "PSICalculator.__init__")
|
|
93
147
|
self.buckets = buckets
|
|
94
148
|
self.equal_freq = equal_freq
|
|
95
149
|
self.min_bin_prop = min_bin_prop
|
|
96
150
|
self.content = content
|
|
97
151
|
self.precision = precision
|
|
98
152
|
self.missing_policy = missing_policy
|
|
153
|
+
self.psi_missing_bucket_policy = psi_missing_bucket_policy
|
|
99
154
|
|
|
100
155
|
# def _calculate_single_psi(
|
|
101
156
|
# self,
|
|
@@ -434,7 +489,8 @@ class PSICalculator:
|
|
|
434
489
|
group_by: Optional[str] = None,
|
|
435
490
|
group_name: Optional[str] = None,
|
|
436
491
|
return_details = False,
|
|
437
|
-
missing_policy: Optional[str] = None
|
|
492
|
+
missing_policy: Optional[str] = None,
|
|
493
|
+
psi_missing_bucket_policy: Optional[str] = None,
|
|
438
494
|
) -> pd.DataFrame:
|
|
439
495
|
"""
|
|
440
496
|
Calculate grouped PSI comparing two datasets, using expected as benchmark.
|
|
@@ -463,6 +519,11 @@ class PSICalculator:
|
|
|
463
519
|
Grouped PSI results.
|
|
464
520
|
"""
|
|
465
521
|
effective_policy = missing_policy if missing_policy is not None else self.missing_policy
|
|
522
|
+
effective_bucket_policy = (
|
|
523
|
+
psi_missing_bucket_policy
|
|
524
|
+
if psi_missing_bucket_policy is not None
|
|
525
|
+
else self.psi_missing_bucket_policy
|
|
526
|
+
)
|
|
466
527
|
return calculate_multigroup_psi_two_sets(
|
|
467
528
|
expected_df = expected_df,
|
|
468
529
|
actual_df = current_data,
|
|
@@ -475,7 +536,8 @@ class PSICalculator:
|
|
|
475
536
|
precision = self.precision,
|
|
476
537
|
group_name = group_name,
|
|
477
538
|
return_details = return_details,
|
|
478
|
-
missing_policy = effective_policy
|
|
539
|
+
missing_policy = effective_policy,
|
|
540
|
+
psi_missing_bucket_policy = effective_bucket_policy,
|
|
479
541
|
)
|
|
480
542
|
|
|
481
543
|
|
|
@@ -493,6 +555,7 @@ def _calculate_single_psi(
|
|
|
493
555
|
content: float = 1e-6,
|
|
494
556
|
precision: int = 5,
|
|
495
557
|
missing_policy: str = "include",
|
|
558
|
+
psi_missing_bucket_policy: str = "smooth_laplace",
|
|
496
559
|
) -> Union[float, Tuple[float, pd.DataFrame]]:
|
|
497
560
|
"""
|
|
498
561
|
Calculate Population Stability Index (PSI) for a single variable.
|
|
@@ -559,6 +622,7 @@ def _calculate_single_psi(
|
|
|
559
622
|
f"_calculate_single_psi: missing_policy must be one of "
|
|
560
623
|
f"'include', 'drop', 'warn_and_drop'; got {missing_policy!r}."
|
|
561
624
|
)
|
|
625
|
+
_validate_psi_bucket_policy(psi_missing_bucket_policy, "_calculate_single_psi")
|
|
562
626
|
|
|
563
627
|
# Record NaN counts before splitting so we can (a) re-attach the missing
|
|
564
628
|
# bin under "include" and (b) warn under "warn_and_drop".
|
|
@@ -631,25 +695,24 @@ def _calculate_single_psi(
|
|
|
631
695
|
)
|
|
632
696
|
expected_denom = float(n_expected) if n_expected > 0 else 1.0
|
|
633
697
|
actual_denom = float(n_actual) if n_actual > 0 else 1.0
|
|
634
|
-
expected_percents = expected_count / expected_denom
|
|
635
|
-
actual_percents = actual_count / actual_denom
|
|
636
698
|
else:
|
|
637
699
|
# "drop" / "warn_and_drop": legacy behaviour, denominators are the
|
|
638
700
|
# NaN-dropped row counts, so missing-rate drift is invisible.
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
701
|
+
expected_denom = float(len(expected_bins)) if len(expected_bins) > 0 else 1.0
|
|
702
|
+
actual_denom = float(len(actual_bins)) if len(actual_bins) > 0 else 1.0
|
|
703
|
+
|
|
704
|
+
all_bins = expected_count.index.union(actual_count.index)
|
|
705
|
+
expected_count = expected_count.reindex(all_bins, fill_value=0).astype(float)
|
|
706
|
+
actual_count = actual_count.reindex(all_bins, fill_value=0).astype(float)
|
|
707
|
+
|
|
708
|
+
expected_percents, actual_percents, psi_values, psi_floor_values = _psi_distributions_from_counts(
|
|
709
|
+
expected_count,
|
|
710
|
+
actual_count,
|
|
711
|
+
expected_denom,
|
|
712
|
+
actual_denom,
|
|
713
|
+
content=content,
|
|
714
|
+
policy=psi_missing_bucket_policy,
|
|
715
|
+
)
|
|
653
716
|
psi_total = psi_values.sum()
|
|
654
717
|
|
|
655
718
|
if return_details:
|
|
@@ -658,8 +721,13 @@ def _calculate_single_psi(
|
|
|
658
721
|
'actual_count': actual_count,
|
|
659
722
|
'expected_percent': expected_percents,
|
|
660
723
|
'actual_percent': actual_percents,
|
|
661
|
-
'psi_component': psi_values
|
|
724
|
+
'psi_component': psi_values,
|
|
725
|
+
'psi_component_floor_1e6': psi_floor_values,
|
|
662
726
|
})
|
|
727
|
+
details["bucket_status"] = "common"
|
|
728
|
+
details.loc[(details["expected_count"] == 0) & (details["actual_count"] > 0), "bucket_status"] = "actual_only"
|
|
729
|
+
details.loc[(details["expected_count"] > 0) & (details["actual_count"] == 0), "bucket_status"] = "expected_only"
|
|
730
|
+
details["psi_missing_bucket_policy"] = psi_missing_bucket_policy
|
|
663
731
|
return psi_total, details
|
|
664
732
|
else:
|
|
665
733
|
return psi_total
|
|
@@ -676,7 +744,8 @@ def calculate_psi(
|
|
|
676
744
|
min_bin_prop: float = 0.05,
|
|
677
745
|
content: float = 1e-6,
|
|
678
746
|
precision: int = 5,
|
|
679
|
-
missing_policy: str = "include"
|
|
747
|
+
missing_policy: str = "include",
|
|
748
|
+
psi_missing_bucket_policy: str = "smooth_laplace",
|
|
680
749
|
) -> Union[float, pd.DataFrame, Tuple[Dict, Dict]]:
|
|
681
750
|
"""
|
|
682
751
|
Calculate Population Stability Index (PSI) for a variable, optionally by groups.
|
|
@@ -767,7 +836,8 @@ def calculate_psi(
|
|
|
767
836
|
min_bin_prop,
|
|
768
837
|
content,
|
|
769
838
|
precision,
|
|
770
|
-
missing_policy
|
|
839
|
+
missing_policy,
|
|
840
|
+
psi_missing_bucket_policy,
|
|
771
841
|
)
|
|
772
842
|
results[group] = psi_value
|
|
773
843
|
details_dict[group] = detail
|
|
@@ -781,7 +851,8 @@ def calculate_psi(
|
|
|
781
851
|
min_bin_prop,
|
|
782
852
|
content,
|
|
783
853
|
precision,
|
|
784
|
-
missing_policy
|
|
854
|
+
missing_policy,
|
|
855
|
+
psi_missing_bucket_policy,
|
|
785
856
|
)
|
|
786
857
|
|
|
787
858
|
if return_details:
|
|
@@ -800,7 +871,8 @@ def calculate_psi(
|
|
|
800
871
|
min_bin_prop,
|
|
801
872
|
content,
|
|
802
873
|
precision,
|
|
803
|
-
missing_policy
|
|
874
|
+
missing_policy,
|
|
875
|
+
psi_missing_bucket_policy,
|
|
804
876
|
)
|
|
805
877
|
else:
|
|
806
878
|
return _calculate_single_psi(
|
|
@@ -812,7 +884,8 @@ def calculate_psi(
|
|
|
812
884
|
min_bin_prop,
|
|
813
885
|
content,
|
|
814
886
|
precision,
|
|
815
|
-
missing_policy
|
|
887
|
+
missing_policy,
|
|
888
|
+
psi_missing_bucket_policy,
|
|
816
889
|
)
|
|
817
890
|
|
|
818
891
|
|
|
@@ -828,7 +901,8 @@ def calculate_within_psi(
|
|
|
828
901
|
content: float = 1e-6,
|
|
829
902
|
precision: int = 5,
|
|
830
903
|
benchmark_display_name: Optional[str] = None,
|
|
831
|
-
missing_policy: str = "include"
|
|
904
|
+
missing_policy: str = "include",
|
|
905
|
+
psi_missing_bucket_policy: str = "smooth_laplace",
|
|
832
906
|
) -> Union[pd.DataFrame, Dict]:
|
|
833
907
|
"""
|
|
834
908
|
Calculate PSI values within a single dataset, comparing groups to a benchmark.
|
|
@@ -904,7 +978,8 @@ def calculate_within_psi(
|
|
|
904
978
|
min_bin_prop=min_bin_prop,
|
|
905
979
|
content=content,
|
|
906
980
|
precision=precision,
|
|
907
|
-
missing_policy=missing_policy
|
|
981
|
+
missing_policy=missing_policy,
|
|
982
|
+
psi_missing_bucket_policy=psi_missing_bucket_policy,
|
|
908
983
|
)
|
|
909
984
|
res_dict[obs_value] = psi
|
|
910
985
|
detail_dict[obs_value] = details
|
|
@@ -919,7 +994,8 @@ def calculate_within_psi(
|
|
|
919
994
|
min_bin_prop=min_bin_prop,
|
|
920
995
|
content=content,
|
|
921
996
|
precision=precision,
|
|
922
|
-
missing_policy=missing_policy
|
|
997
|
+
missing_policy=missing_policy,
|
|
998
|
+
psi_missing_bucket_policy=psi_missing_bucket_policy,
|
|
923
999
|
)
|
|
924
1000
|
res_dict[obs_value] = psi
|
|
925
1001
|
|
|
@@ -943,7 +1019,8 @@ def calculate_psi_within_dataset(
|
|
|
943
1019
|
min_bin_prop: float = 0.05,
|
|
944
1020
|
content: float = 1e-6,
|
|
945
1021
|
precision: int = 5,
|
|
946
|
-
missing_policy: str = "include"
|
|
1022
|
+
missing_policy: str = "include",
|
|
1023
|
+
psi_missing_bucket_policy: str = "smooth_laplace",
|
|
947
1024
|
) -> pd.DataFrame:
|
|
948
1025
|
"""
|
|
949
1026
|
Calculate PSI for multiple variables within a dataset, comparing groups to a benchmark.
|
|
@@ -1003,7 +1080,8 @@ def calculate_psi_within_dataset(
|
|
|
1003
1080
|
min_bin_prop=min_bin_prop,
|
|
1004
1081
|
content=content,
|
|
1005
1082
|
precision=precision,
|
|
1006
|
-
missing_policy=missing_policy
|
|
1083
|
+
missing_policy=missing_policy,
|
|
1084
|
+
psi_missing_bucket_policy=psi_missing_bucket_policy,
|
|
1007
1085
|
).sort_values([grp_name]).reset_index(drop=True)
|
|
1008
1086
|
|
|
1009
1087
|
single_psi['var'] = var
|
|
@@ -1022,7 +1100,8 @@ def calculate_multivar_psi_two_sets(
|
|
|
1022
1100
|
min_bin_prop: float = 0.05,
|
|
1023
1101
|
content: float = 1e-6,
|
|
1024
1102
|
precision: int = 5,
|
|
1025
|
-
missing_policy: str = "include"
|
|
1103
|
+
missing_policy: str = "include",
|
|
1104
|
+
psi_missing_bucket_policy: str = "smooth_laplace",
|
|
1026
1105
|
) -> pd.DataFrame:
|
|
1027
1106
|
"""
|
|
1028
1107
|
Calculate PSI for multiple variables by comparing two different datasets.
|
|
@@ -1077,8 +1156,13 @@ def calculate_multivar_psi_two_sets(
|
|
|
1077
1156
|
target_col=var,
|
|
1078
1157
|
group_by=group_by,
|
|
1079
1158
|
buckets=buckets,
|
|
1159
|
+
equal_freq=equal_freq,
|
|
1160
|
+
min_bin_prop=min_bin_prop,
|
|
1161
|
+
content=content,
|
|
1162
|
+
precision=precision,
|
|
1080
1163
|
return_details=False,
|
|
1081
|
-
missing_policy=missing_policy
|
|
1164
|
+
missing_policy=missing_policy,
|
|
1165
|
+
psi_missing_bucket_policy=psi_missing_bucket_policy,
|
|
1082
1166
|
)
|
|
1083
1167
|
if group_by is None:
|
|
1084
1168
|
single_psi = pd.DataFrame([single_psi], columns=['psi'])
|
|
@@ -1219,7 +1303,8 @@ def calculate_multigroup_psi_two_sets(
|
|
|
1219
1303
|
precision: int = 5,
|
|
1220
1304
|
group_name: Optional[str] = None,
|
|
1221
1305
|
return_details: bool = False,
|
|
1222
|
-
missing_policy: str = "include"
|
|
1306
|
+
missing_policy: str = "include",
|
|
1307
|
+
psi_missing_bucket_policy: str = "smooth_laplace",
|
|
1223
1308
|
) -> Union[pd.DataFrame, Dict[str, pd.DataFrame]]:
|
|
1224
1309
|
"""
|
|
1225
1310
|
Calculate grouped PSI using expected DataFrame as benchmark, applied to actual DataFrame groups.
|
|
@@ -1278,7 +1363,8 @@ def calculate_multigroup_psi_two_sets(
|
|
|
1278
1363
|
min_bin_prop=min_bin_prop,
|
|
1279
1364
|
content=content,
|
|
1280
1365
|
precision=precision,
|
|
1281
|
-
missing_policy=missing_policy
|
|
1366
|
+
missing_policy=missing_policy,
|
|
1367
|
+
psi_missing_bucket_policy=psi_missing_bucket_policy,
|
|
1282
1368
|
)
|
|
1283
1369
|
|
|
1284
1370
|
psi_records.append({group_name: group, 'var': var, 'psi': psi_val})
|
|
@@ -1296,7 +1382,7 @@ def calculate_multigroup_psi_two_sets(
|
|
|
1296
1382
|
detail_df[group_name] = group
|
|
1297
1383
|
detail_df['var'] = var
|
|
1298
1384
|
|
|
1299
|
-
required_cols = ['bin', 'expected_count', 'actual_count', 'expected_percent', 'actual_percent', 'psi_component', group_name, 'var']
|
|
1385
|
+
required_cols = ['bin', 'expected_count', 'actual_count', 'expected_percent', 'actual_percent', 'psi_component', 'bucket_status', 'psi_missing_bucket_policy', group_name, 'var']
|
|
1300
1386
|
for col in required_cols:
|
|
1301
1387
|
if col not in detail_df.columns:
|
|
1302
1388
|
detail_df[col] = np.nan
|
|
@@ -1326,7 +1412,8 @@ def calculate_multigroup_psi_two_sets(
|
|
|
1326
1412
|
min_bin_prop=min_bin_prop,
|
|
1327
1413
|
content=content,
|
|
1328
1414
|
precision=precision,
|
|
1329
|
-
missing_policy=missing_policy
|
|
1415
|
+
missing_policy=missing_policy,
|
|
1416
|
+
psi_missing_bucket_policy=psi_missing_bucket_policy,
|
|
1330
1417
|
)
|
|
1331
1418
|
psi_records.append({'var': var, 'psi': psi_val})
|
|
1332
1419
|
|
|
@@ -1338,7 +1425,7 @@ def calculate_multigroup_psi_two_sets(
|
|
|
1338
1425
|
index_col = detail_df.columns[0]
|
|
1339
1426
|
detail_df = detail_df.rename(columns={index_col: 'bin'})
|
|
1340
1427
|
detail_df['var'] = var
|
|
1341
|
-
required_cols = ['bin', 'expected_count', 'actual_count', 'expected_percent', 'actual_percent', 'psi_component', 'var']
|
|
1428
|
+
required_cols = ['bin', 'expected_count', 'actual_count', 'expected_percent', 'actual_percent', 'psi_component', 'bucket_status', 'psi_missing_bucket_policy', 'var']
|
|
1342
1429
|
for col in required_cols:
|
|
1343
1430
|
if col not in detail_df.columns:
|
|
1344
1431
|
detail_df[col] = np.nan
|
|
@@ -1362,7 +1449,8 @@ def calculate_multigroup_psi_two_sets(
|
|
|
1362
1449
|
min_bin_prop=min_bin_prop,
|
|
1363
1450
|
content=content,
|
|
1364
1451
|
precision=precision,
|
|
1365
|
-
missing_policy=missing_policy
|
|
1452
|
+
missing_policy=missing_policy,
|
|
1453
|
+
psi_missing_bucket_policy=psi_missing_bucket_policy,
|
|
1366
1454
|
)
|
|
1367
1455
|
|
|
1368
1456
|
return group_psi
|
{supermodelingfactory-0.5.0 → supermodelingfactory-0.5.2}/Modeling_Tool/Feature/PSI_Tool.pyi
RENAMED
|
@@ -20,10 +20,10 @@ from tqdm import tqdm
|
|
|
20
20
|
from Modeling_Tool.Core.Binning_Tool import quick_binning
|
|
21
21
|
|
|
22
22
|
class PSICalculator:
|
|
23
|
-
def __init__(self, buckets: int = 10, equal_freq: bool = True, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, binning_engine: Any = None): ...
|
|
24
|
-
def calculate(self, expected_df: pd.DataFrame, current_data: pd.DataFrame, varlist: List[str], group_by: Optional[str] = None, group_name: Optional[str] = None, return_details = False) -> pd.DataFrame: ...
|
|
25
|
-
def calculate_psi(expected: Union[pd.DataFrame, pd.Series], actual: Union[pd.DataFrame, pd.Series], target_col: str, buckets: int = 10, equal_freq: bool = True, group_by: Optional[Union[str, List[str]]] = None, return_details: bool = False, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5) -> Union[float, pd.DataFrame, Tuple[Dict, Dict]]: ...
|
|
26
|
-
def calculate_within_psi(data: pd.DataFrame, grp_name: str, target_col: str, benchmark: Optional[Any] = None, equal_freq: bool = True, buckets: int = 10, return_details: bool = False, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, benchmark_display_name: Optional[str] = None) -> Union[pd.DataFrame, Dict]: ...
|
|
27
|
-
def calculate_psi_within_dataset(data: pd.DataFrame, grp_name: str, varlist: List[str], benchmark: Optional[Any] = None, equal_freq: bool = True, buckets: int = 10, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5) -> pd.DataFrame: ...
|
|
28
|
-
def calculate_multivar_psi_two_sets(expected_df: pd.DataFrame, actual_df: pd.DataFrame, varlist: List[str], group_by: Optional[Union[str, List[str]]] = None, buckets: int = 10, equal_freq: bool = True, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5) -> pd.DataFrame: ...
|
|
29
|
-
def calculate_multigroup_psi_two_sets(expected_df: pd.DataFrame, actual_df: pd.DataFrame, varlist: List[str], group_by: Optional[Union[str, List[str]]] = None, buckets: int = 10, equal_freq: bool = True, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, group_name: Optional[str] = None, return_details: bool = False) -> Union[pd.DataFrame, Dict[str, pd.DataFrame]]: ...
|
|
23
|
+
def __init__(self, buckets: int = 10, equal_freq: bool = True, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, missing_policy: str = "include", psi_missing_bucket_policy: str = "smooth_laplace", binning_engine: Any = None): ...
|
|
24
|
+
def calculate(self, expected_df: pd.DataFrame, current_data: pd.DataFrame, varlist: List[str], group_by: Optional[str] = None, group_name: Optional[str] = None, return_details = False, missing_policy: Optional[str] = None, psi_missing_bucket_policy: Optional[str] = None) -> pd.DataFrame: ...
|
|
25
|
+
def calculate_psi(expected: Union[pd.DataFrame, pd.Series], actual: Union[pd.DataFrame, pd.Series], target_col: str, buckets: int = 10, equal_freq: bool = True, group_by: Optional[Union[str, List[str]]] = None, return_details: bool = False, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, missing_policy: str = "include", psi_missing_bucket_policy: str = "smooth_laplace") -> Union[float, pd.DataFrame, Tuple[Dict, Dict]]: ...
|
|
26
|
+
def calculate_within_psi(data: pd.DataFrame, grp_name: str, target_col: str, benchmark: Optional[Any] = None, equal_freq: bool = True, buckets: int = 10, return_details: bool = False, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, benchmark_display_name: Optional[str] = None, missing_policy: str = "include", psi_missing_bucket_policy: str = "smooth_laplace") -> Union[pd.DataFrame, Dict]: ...
|
|
27
|
+
def calculate_psi_within_dataset(data: pd.DataFrame, grp_name: str, varlist: List[str], benchmark: Optional[Any] = None, equal_freq: bool = True, buckets: int = 10, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, missing_policy: str = "include", psi_missing_bucket_policy: str = "smooth_laplace") -> pd.DataFrame: ...
|
|
28
|
+
def calculate_multivar_psi_two_sets(expected_df: pd.DataFrame, actual_df: pd.DataFrame, varlist: List[str], group_by: Optional[Union[str, List[str]]] = None, buckets: int = 10, equal_freq: bool = True, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, missing_policy: str = "include", psi_missing_bucket_policy: str = "smooth_laplace") -> pd.DataFrame: ...
|
|
29
|
+
def calculate_multigroup_psi_two_sets(expected_df: pd.DataFrame, actual_df: pd.DataFrame, varlist: List[str], group_by: Optional[Union[str, List[str]]] = None, buckets: int = 10, equal_freq: bool = True, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, group_name: Optional[str] = None, return_details: bool = False, missing_policy: str = "include", psi_missing_bucket_policy: str = "smooth_laplace") -> Union[pd.DataFrame, Dict[str, pd.DataFrame]]: ...
|