SuperModelingFactory 0.5.4__tar.gz → 0.5.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/Parallel_ODPS_Manager.py +221 -50
  2. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Pipeline/feature_validation.py +21 -1
  3. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Pipeline/field_meta.py +9 -2
  4. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/__init__.py +1 -1
  5. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/PKG-INFO +2 -2
  6. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/README.md +1 -1
  7. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/SuperModelingFactory.egg-info/PKG-INFO +2 -2
  8. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/pyproject.toml +1 -1
  9. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/setup.py +1 -1
  10. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/ExcelMaster/ExcelFormatTool.py +0 -0
  11. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/ExcelMaster/ExcelMaster.py +0 -0
  12. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/ExcelMaster/Template.py +0 -0
  13. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/ExcelMaster/Utility.py +0 -0
  14. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/ExcelMaster/__init__.py +0 -0
  15. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/LICENSE +0 -0
  16. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/MANIFEST.in +0 -0
  17. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/Binning_Tool.py +0 -0
  18. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/Binning_Tool.pyi +0 -0
  19. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/Check_DuckDB_Compatibility.py +0 -0
  20. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/Json_Data_Converter.py +0 -0
  21. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/Model_Registry_Tool.py +0 -0
  22. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/ODPS_Tool.py +0 -0
  23. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/Parallel_Engine.py +0 -0
  24. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/Proc_Compare.py +0 -0
  25. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/Slope_Tool.py +0 -0
  26. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/Slope_Tool.pyi +0 -0
  27. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/XOR_Encryptor.py +0 -0
  28. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/XOR_Encryptor.pyi +0 -0
  29. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/__init__.py +0 -0
  30. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/kDataFrame.py +0 -0
  31. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/kDataFrame.pyi +0 -0
  32. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/sample_weight_utils.py +0 -0
  33. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Core/utils.py +0 -0
  34. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Eval/Evaluation_Tool.py +0 -0
  35. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Eval/Evaluation_Tool.pyi +0 -0
  36. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Eval/Model_Eval_Tool.py +0 -0
  37. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Eval/Model_Eval_Tool.pyi +0 -0
  38. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Eval/__init__.py +0 -0
  39. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Eval/evaluate_model.py +0 -0
  40. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Eval/evaluate_model.pyi +0 -0
  41. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Eval/weighted_eval_utils.py +0 -0
  42. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Explainability/Coalition_Structure.py +0 -0
  43. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Explainability/Model_Explainer.py +0 -0
  44. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Explainability/__init__.py +0 -0
  45. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Feature/Distribution_Tool.py +0 -0
  46. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Feature/Distribution_Tool.pyi +0 -0
  47. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Feature/Feature_Insights.py +0 -0
  48. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Feature/Feature_Insights.pyi +0 -0
  49. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Feature/Feature_Screen.py +0 -0
  50. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Feature/PSI_Tool.py +0 -0
  51. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Feature/PSI_Tool.pyi +0 -0
  52. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py +0 -0
  53. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Feature/Weighted_Screen.py +0 -0
  54. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Feature/__init__.py +0 -0
  55. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Model/Backward_Tool.py +0 -0
  56. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Model/Backward_Tool.pyi +0 -0
  57. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Model/GBM_Search_Tool.py +0 -0
  58. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Model/GBM_Tool.py +0 -0
  59. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Model/GBM_Tool.pyi +0 -0
  60. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Model/LRM_Tool.py +0 -0
  61. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Model/LRM_Tool.pyi +0 -0
  62. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Model/__init__.py +0 -0
  63. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Pipeline/__init__.py +0 -0
  64. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Pipeline/_common.py +0 -0
  65. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Pipeline/credit_model.py +0 -0
  66. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Pipeline/mock_sample.py +0 -0
  67. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Pipeline/orchestrator.py +0 -0
  68. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Pipeline/reject_inference.py +0 -0
  69. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Pipeline/sample_analysis.py +0 -0
  70. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Pipeline/score_comparison.py +0 -0
  71. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Pipeline/score_consistency_uat.py +0 -0
  72. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Pipeline/screening_artifact.py +0 -0
  73. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Sample/Distribution_Adaptation.py +0 -0
  74. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Sample/Distribution_Adaptation.pyi +0 -0
  75. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Sample/Reject_Infer.py +0 -0
  76. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Sample/Reject_Infer.pyi +0 -0
  77. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Sample/Sample_Split.py +0 -0
  78. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Sample/Sample_Split.pyi +0 -0
  79. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/Sample/__init__.py +0 -0
  80. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/UAT/UAT_Consistency_Checker.py +0 -0
  81. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/UAT/__init__.py +0 -0
  82. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/WOE_Adapter.py +0 -0
  83. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/WOE_Adapter.pyi +0 -0
  84. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/WOE_Master.py +0 -0
  85. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/WOE_Master.pyi +0 -0
  86. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/WOE_Monotone_Binner.py +0 -0
  87. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/WOE_Monotone_Binner.pyi +0 -0
  88. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/WOE_Plot_Tool.py +0 -0
  89. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/WOE_Plot_Tool.pyi +0 -0
  90. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/WOE_Report_Builder.py +0 -0
  91. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/WOE_Report_Builder.pyi +0 -0
  92. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/WOE_Tool.py +0 -0
  93. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/WOE_Tool.pyi +0 -0
  94. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/__init__.py +0 -0
  95. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/plot_woe_tool.py +0 -0
  96. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/WOE/plot_woe_tool.pyi +0 -0
  97. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/_utils/__init__.py +0 -0
  98. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/_utils/nan_guard.py +0 -0
  99. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/_utils/robust.py +0 -0
  100. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/_utils/sentinels.py +0 -0
  101. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/ref_font/KaiTi.ttf +0 -0
  102. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/ref_font/WeiRuanYaHei.ttf +0 -0
  103. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/ref_font/__init__.py +0 -0
  104. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Modeling_Tool/ref_font/simsun.ttc +0 -0
  105. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Report/Report_Tool.py +0 -0
  106. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/Report/__init__.py +0 -0
  107. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/SuperModelingFactory.egg-info/SOURCES.txt +0 -0
  108. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/SuperModelingFactory.egg-info/dependency_links.txt +0 -0
  109. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/SuperModelingFactory.egg-info/not-zip-safe +0 -0
  110. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/SuperModelingFactory.egg-info/requires.txt +0 -0
  111. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/SuperModelingFactory.egg-info/top_level.txt +0 -0
  112. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/requirements.txt +0 -0
  113. {supermodelingfactory-0.5.4 → supermodelingfactory-0.5.5}/setup.cfg +0 -0
@@ -17,6 +17,7 @@ from .utils import parse_sql_file
17
17
 
18
18
  Backend = Literal["thread", "process", "sequential"]
19
19
  WriteMode = Literal["overwrite", "append"]
20
+ PullSplitStrategy = Literal["auto", "hash", "row_number"]
20
21
 
21
22
 
22
23
  @dataclass
@@ -26,6 +27,10 @@ class ParallelODPSConfig:
26
27
  n_chunks: int | None = None
27
28
  n_jobs: int = 3
28
29
  backend: Backend = "thread"
30
+ pull_split_strategy: PullSplitStrategy = "auto"
31
+ row_number_order_by: str | None = None
32
+ row_number_col: str = "__smf_parallel_odps_rn__"
33
+ validate_unique_key: bool = True
29
34
  chunk_filter_key: str = "chunk_filter"
30
35
  tmp_dir: Path = field(default_factory=lambda: Path("data/_chunks"))
31
36
  tmp_table_prefix: str = "tmp_parallel_odps"
@@ -55,22 +60,23 @@ def _delete_table(runner: ODPSRunner, table_name: str) -> None:
55
60
  runner.run_sql(f"DROP TABLE IF EXISTS {table_name};", to_df=False)
56
61
 
57
62
 
63
+ def _strip_trailing_semicolon(sql: str) -> str:
64
+ return sql.strip().rstrip(";").strip()
65
+
66
+
58
67
  def _pull_one_chunk(
59
- chunk_id: int,
68
+ chunk_spec: dict[str, Any],
60
69
  runner: ODPSRunner | None,
61
- sql_path: str,
62
- template_kwargs: dict[str, Any],
63
- chunk_filter_key: str,
64
- unique_key: str,
65
- n_chunks: int,
66
70
  tmp_dir_str: str,
67
71
  ) -> dict[str, Any]:
68
72
  active_runner = _get_runner(runner)
69
- kwargs = dict(template_kwargs)
70
- kwargs[chunk_filter_key] = f"ABS(HASH({unique_key})) % {n_chunks} = {chunk_id}"
71
- rendered_sql = parse_sql_file(sql_path=sql_path, **kwargs)
73
+ chunk_id = int(chunk_spec["chunk"])
74
+ rendered_sql = str(chunk_spec["sql"])
75
+ drop_cols = list(chunk_spec.get("drop_cols") or [])
72
76
 
73
77
  df_chunk = active_runner.run_sql(rendered_sql, to_df=True, n_process=1)
78
+ if drop_cols:
79
+ df_chunk = df_chunk.drop(columns=[col for col in drop_cols if col in df_chunk.columns])
74
80
  chunk_path = Path(tmp_dir_str) / f"_pull_chunk_{chunk_id:04d}.csv"
75
81
  df_chunk.to_csv(chunk_path, index=False)
76
82
  n_rows = len(df_chunk)
@@ -102,6 +108,7 @@ class ParallelODPSManager:
102
108
 
103
109
  _VALID_BACKENDS = {"thread", "process", "sequential"}
104
110
  _VALID_WRITE_MODES = {"overwrite", "append"}
111
+ _VALID_PULL_STRATEGIES = {"auto", "hash", "row_number"}
105
112
 
106
113
  def __init__(self, config: ParallelODPSConfig, odps_runner: ODPSRunner | None = None):
107
114
  self.config = config
@@ -112,6 +119,8 @@ class ParallelODPSManager:
112
119
  cfg = self.config
113
120
  if cfg.backend not in self._VALID_BACKENDS:
114
121
  raise ValueError(f"backend must be one of {sorted(self._VALID_BACKENDS)}")
122
+ if cfg.pull_split_strategy not in self._VALID_PULL_STRATEGIES:
123
+ raise ValueError(f"pull_split_strategy must be one of {sorted(self._VALID_PULL_STRATEGIES)}")
115
124
  if cfg.chunk_size is not None and cfg.chunk_size <= 0:
116
125
  raise ValueError("chunk_size must be a positive integer.")
117
126
  if cfg.n_chunks is not None and cfg.n_chunks <= 0:
@@ -122,6 +131,8 @@ class ParallelODPSManager:
122
131
  raise ValueError("n_jobs must be a positive integer.")
123
132
  if not cfg.chunk_filter_key:
124
133
  raise ValueError("chunk_filter_key cannot be empty.")
134
+ if not cfg.row_number_col:
135
+ raise ValueError("row_number_col cannot be empty.")
125
136
  if not cfg.tmp_table_prefix:
126
137
  raise ValueError("tmp_table_prefix cannot be empty.")
127
138
 
@@ -131,34 +142,151 @@ class ParallelODPSManager:
131
142
  def _auto_count_query(self, sql_path: str, template_kwargs: dict[str, Any]) -> str:
132
143
  kwargs = dict(template_kwargs)
133
144
  kwargs[self.config.chunk_filter_key] = "1=1"
134
- rendered_sql = parse_sql_file(sql_path=sql_path, **kwargs).rstrip().rstrip(";")
145
+ rendered_sql = _strip_trailing_semicolon(parse_sql_file(sql_path=sql_path, **kwargs))
135
146
  return f"SELECT COUNT(1) FROM ({rendered_sql}) __count_src;"
136
147
 
137
- def _resolve_pull_n_chunks(self, count_query: str | None, sql_path: str, template_kwargs: dict[str, Any]) -> int:
148
+ def _assert_chunk_filter_placeholder(self, sql_path: str, template_kwargs: dict[str, Any]) -> None:
149
+ sentinel = "__SMF_CHUNK_FILTER_SENTINEL__=1"
150
+ kwargs = dict(template_kwargs)
151
+ kwargs[self.config.chunk_filter_key] = sentinel
152
+ rendered_sql = parse_sql_file(sql_path=sql_path, **kwargs)
153
+ if sentinel not in rendered_sql:
154
+ raise ValueError(
155
+ f"pull SQL template must contain {{{self.config.chunk_filter_key}}}"
156
+ )
157
+
158
+ def _render_pull_sql(self, sql_path: str, template_kwargs: dict[str, Any], chunk_filter: str) -> str:
159
+ kwargs = dict(template_kwargs)
160
+ kwargs[self.config.chunk_filter_key] = chunk_filter
161
+ return parse_sql_file(sql_path=sql_path, **kwargs)
162
+
163
+ def _resolve_pull_strategy(self) -> str:
164
+ cfg = self.config
165
+ strategy = cfg.pull_split_strategy
166
+ if strategy == "auto":
167
+ return "hash" if cfg.unique_key else "row_number"
168
+ if strategy == "hash" and not cfg.unique_key:
169
+ raise ValueError("unique_key is required when pull_split_strategy='hash'.")
170
+ return strategy
171
+
172
+ def _resolve_pull_n_chunks(
173
+ self,
174
+ count_query: str | None,
175
+ sql_path: str,
176
+ template_kwargs: dict[str, Any],
177
+ strategy: str,
178
+ staging_table: str | None = None,
179
+ ) -> int:
138
180
  cfg = self.config
139
- if not cfg.unique_key:
140
- raise ValueError("unique_key is required for pull().")
141
181
  if cfg.n_chunks is not None:
142
182
  return cfg.n_chunks
143
183
  if cfg.chunk_size is None:
144
184
  raise ValueError("chunk_size or n_chunks is required for pull().")
145
185
  if count_query is None:
146
- count_query = self._auto_count_query(sql_path, template_kwargs)
186
+ if strategy == "row_number":
187
+ if not staging_table:
188
+ raise ValueError("staging_table is required to count row_number pull chunks.")
189
+ count_query = f"SELECT COUNT(1) FROM {staging_table};"
190
+ else:
191
+ count_query = self._auto_count_query(sql_path, template_kwargs)
147
192
  count_df = self.odps_runner.run_sql(count_query, to_df=True)
148
193
  total_rows = int(count_df.iloc[0, 0])
149
194
  return max(1, math.ceil(total_rows / cfg.chunk_size))
150
195
 
151
- def pull(
196
+ def _hash_chunk_filter(self, chunk_id: int, n_chunks: int) -> str:
197
+ return f"ABS(HASH({self.config.unique_key})) % {n_chunks} = {chunk_id}"
198
+
199
+ def _validate_hash_pull_sql(
152
200
  self,
153
201
  sql_path: str,
154
- out_path: str,
155
- count_query: str | None = None,
156
- **template_kwargs: Any,
157
- ) -> dict[str, Any]:
202
+ template_kwargs: dict[str, Any],
203
+ n_chunks: int,
204
+ ) -> None:
205
+ if not self.config.validate_unique_key:
206
+ return
207
+ rendered_sql = self._render_pull_sql(
208
+ sql_path=sql_path,
209
+ template_kwargs=template_kwargs,
210
+ chunk_filter=self._hash_chunk_filter(0, n_chunks),
211
+ )
212
+ probe_sql = (
213
+ "SELECT * FROM (\n"
214
+ f"{_strip_trailing_semicolon(rendered_sql)}\n"
215
+ ") __smf_unique_key_probe LIMIT 1;"
216
+ )
217
+ try:
218
+ self.odps_runner.run_sql(probe_sql, to_df=True, n_process=1)
219
+ except Exception as exc:
220
+ raise ValueError(
221
+ f"unique_key validation failed for pull SQL. "
222
+ f"Check that unique_key={self.config.unique_key!r} is visible in the SQL scope."
223
+ ) from exc
224
+
225
+ def _build_hash_pull_chunks(
226
+ self,
227
+ sql_path: str,
228
+ template_kwargs: dict[str, Any],
229
+ n_chunks: int,
230
+ ) -> list[dict[str, Any]]:
231
+ return [
232
+ {
233
+ "chunk": chunk_id,
234
+ "sql": self._render_pull_sql(
235
+ sql_path=sql_path,
236
+ template_kwargs=template_kwargs,
237
+ chunk_filter=self._hash_chunk_filter(chunk_id, n_chunks),
238
+ ),
239
+ }
240
+ for chunk_id in range(n_chunks)
241
+ ]
242
+
243
+ def _row_number_staging_table_name(self, run_id: str) -> str:
244
+ return f"{self.config.tmp_table_prefix}_{run_id}_pull_stage"
245
+
246
+ def _create_row_number_staging_table(
247
+ self,
248
+ sql_path: str,
249
+ template_kwargs: dict[str, Any],
250
+ staging_table: str,
251
+ ) -> str:
158
252
  cfg = self.config
159
- n_chunks = self._resolve_pull_n_chunks(count_query, sql_path, template_kwargs)
160
- cfg.tmp_dir.mkdir(parents=True, exist_ok=True)
253
+ base_sql = _strip_trailing_semicolon(
254
+ self._render_pull_sql(
255
+ sql_path=sql_path,
256
+ template_kwargs=template_kwargs,
257
+ chunk_filter="1=1",
258
+ )
259
+ )
260
+ order_by = cfg.row_number_order_by or "1"
261
+ create_sql = (
262
+ f"CREATE TABLE {staging_table} AS\n"
263
+ "SELECT\n"
264
+ f" ROW_NUMBER() OVER (ORDER BY {order_by}) AS {cfg.row_number_col},\n"
265
+ " __smf_base.*\n"
266
+ "FROM (\n"
267
+ f"{base_sql}\n"
268
+ ") __smf_base;"
269
+ )
270
+ self.odps_runner.run_sql(create_sql, to_df=False)
271
+ return create_sql
272
+
273
+ def _build_row_number_pull_chunks(self, staging_table: str, n_chunks: int) -> list[dict[str, Any]]:
274
+ row_number_col = self.config.row_number_col
275
+ return [
276
+ {
277
+ "chunk": chunk_id,
278
+ "sql": (
279
+ f"SELECT *\n"
280
+ f"FROM {staging_table}\n"
281
+ f"WHERE ({row_number_col} - 1) % {n_chunks} = {chunk_id};"
282
+ ),
283
+ "drop_cols": [row_number_col],
284
+ }
285
+ for chunk_id in range(n_chunks)
286
+ ]
161
287
 
288
+ def _run_pull_chunks(self, chunk_specs: list[dict[str, Any]]) -> list[dict[str, Any]]:
289
+ cfg = self.config
162
290
  engine_cfg = ParallelApplyConfig(
163
291
  split_axis="chunk",
164
292
  backend=cfg.backend,
@@ -168,41 +296,83 @@ class ParallelODPSManager:
168
296
  )
169
297
  result = ParallelApplyEngine(engine_cfg).run(
170
298
  func=_pull_one_chunk,
171
- chunks=list(range(n_chunks)),
299
+ chunks=chunk_specs,
172
300
  func_args=(
173
301
  self._runner_for_backend(),
174
- sql_path,
175
- template_kwargs,
176
- cfg.chunk_filter_key,
177
- cfg.unique_key,
178
- n_chunks,
179
302
  str(cfg.tmp_dir),
180
303
  ),
181
304
  )
182
-
183
305
  if len(result.errors):
184
- raise RuntimeError(f"{len(result.errors)}/{n_chunks} ODPS pull chunks failed:\n{result.errors}")
185
-
186
- chunk_summaries = sorted(result.output, key=lambda item: item["chunk"])
187
- final_path = Path(out_path)
188
- final_path.parent.mkdir(parents=True, exist_ok=True)
189
- final_path.unlink(missing_ok=True)
190
-
191
- with open(final_path, "wb") as fout:
192
- for idx, summary in enumerate(chunk_summaries):
193
- chunk_file = Path(summary["path"])
194
- with open(chunk_file, "rb") as fin:
195
- if idx > 0:
196
- fin.readline()
197
- shutil.copyfileobj(fin, fout)
198
- chunk_file.unlink()
199
-
200
- return {
201
- "n_chunks": n_chunks,
202
- "total_rows": sum(item["rows"] for item in chunk_summaries),
203
- "out_path": str(final_path),
204
- "per_chunk_rows": [item["rows"] for item in chunk_summaries],
205
- }
306
+ raise RuntimeError(f"{len(result.errors)}/{len(chunk_specs)} ODPS pull chunks failed:\n{result.errors}")
307
+ return sorted(result.output, key=lambda item: item["chunk"])
308
+
309
+ def pull(
310
+ self,
311
+ sql_path: str,
312
+ out_path: str,
313
+ count_query: str | None = None,
314
+ **template_kwargs: Any,
315
+ ) -> dict[str, Any]:
316
+ cfg = self.config
317
+ self._assert_chunk_filter_placeholder(sql_path, template_kwargs)
318
+ cfg.tmp_dir.mkdir(parents=True, exist_ok=True)
319
+ strategy = self._resolve_pull_strategy()
320
+ staging_table: str | None = None
321
+ success = False
322
+
323
+ try:
324
+ if strategy == "row_number":
325
+ run_id = uuid.uuid4().hex[:12]
326
+ staging_table = self._row_number_staging_table_name(run_id)
327
+ self._create_row_number_staging_table(sql_path, template_kwargs, staging_table)
328
+ n_chunks = self._resolve_pull_n_chunks(
329
+ count_query=count_query,
330
+ sql_path=sql_path,
331
+ template_kwargs=template_kwargs,
332
+ strategy=strategy,
333
+ staging_table=staging_table,
334
+ )
335
+ chunk_specs = self._build_row_number_pull_chunks(staging_table, n_chunks)
336
+ else:
337
+ n_chunks = self._resolve_pull_n_chunks(
338
+ count_query=count_query,
339
+ sql_path=sql_path,
340
+ template_kwargs=template_kwargs,
341
+ strategy=strategy,
342
+ )
343
+ self._validate_hash_pull_sql(sql_path, template_kwargs, n_chunks)
344
+ chunk_specs = self._build_hash_pull_chunks(sql_path, template_kwargs, n_chunks)
345
+
346
+ chunk_summaries = self._run_pull_chunks(chunk_specs)
347
+ final_path = Path(out_path)
348
+ final_path.parent.mkdir(parents=True, exist_ok=True)
349
+ final_path.unlink(missing_ok=True)
350
+
351
+ with open(final_path, "wb") as fout:
352
+ for idx, summary in enumerate(chunk_summaries):
353
+ chunk_file = Path(summary["path"])
354
+ with open(chunk_file, "rb") as fin:
355
+ if idx > 0:
356
+ fin.readline()
357
+ shutil.copyfileobj(fin, fout)
358
+ chunk_file.unlink()
359
+
360
+ success = True
361
+ return {
362
+ "pull_strategy": strategy,
363
+ "staging_table": staging_table,
364
+ "n_chunks": n_chunks,
365
+ "total_rows": sum(item["rows"] for item in chunk_summaries),
366
+ "out_path": str(final_path),
367
+ "per_chunk_rows": [item["rows"] for item in chunk_summaries],
368
+ }
369
+ finally:
370
+ if (
371
+ staging_table
372
+ and cfg.cleanup_tmp
373
+ and (success or not cfg.keep_tmp_on_error)
374
+ ):
375
+ _delete_table(self.odps_runner, staging_table)
206
376
 
207
377
  def push(
208
378
  self,
@@ -223,6 +393,7 @@ class ParallelODPSManager:
223
393
  raise ValueError("push data produced no chunks.")
224
394
 
225
395
  tmp_tables = [spec["tmp_table"] for spec in chunk_specs]
396
+ success = False
226
397
  try:
227
398
  engine_cfg = ParallelApplyConfig(
228
399
  split_axis="chunk",
@@ -2,6 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  import gc
4
4
  import logging
5
+ import warnings
5
6
  from dataclasses import dataclass, field, replace
6
7
  from pathlib import Path
7
8
  from typing import Any, Literal
@@ -33,6 +34,7 @@ class FeatureValidationPipelineConfig:
33
34
 
34
35
  input_type: Literal["auto", "dataframe", "csv"] = "auto"
35
36
  csv_read_kwargs: dict[str, Any] = field(default_factory=dict)
37
+ enable_batch: bool = False
36
38
  feature_batch_size: int | None = None
37
39
  feature_batches: list[list[str]] | None = None
38
40
  batch_base_cols: list[str] | None = None
@@ -150,10 +152,19 @@ class FeatureValidationPipeline:
150
152
 
151
153
  def run(self, data: pd.DataFrame | str | Path) -> FeatureValidationPipelineResult:
152
154
  input_type = self._resolve_input_type(data)
155
+ if self.config.enable_batch and input_type != "csv":
156
+ raise ValueError("enable_batch=True currently requires CSV input.")
153
157
  if input_type == "csv":
154
158
  csv_path = Path(data)
155
159
  if self._is_batch_mode():
156
160
  return self._run_csv_batches(csv_path)
161
+ if self._has_batch_config():
162
+ warnings.warn(
163
+ "feature_batch_size/feature_batches are configured but enable_batch=False; "
164
+ "FeatureValidationPipeline will read the CSV in full and ignore batch settings.",
165
+ RuntimeWarning,
166
+ stacklevel=2,
167
+ )
157
168
  return self._run_dataframe(self._read_csv(csv_path))
158
169
  if not isinstance(data, pd.DataFrame):
159
170
  raise TypeError("data must be a pandas DataFrame when input_type='dataframe'.")
@@ -268,10 +279,18 @@ class FeatureValidationPipeline:
268
279
  return cfg.input_type
269
280
  return "dataframe" if isinstance(data, pd.DataFrame) else "csv"
270
281
 
271
- def _is_batch_mode(self) -> bool:
282
+ def _has_batch_config(self) -> bool:
272
283
  cfg = self.config
273
284
  return bool(cfg.feature_batches) or cfg.feature_batch_size is not None
274
285
 
286
+ def _is_batch_mode(self) -> bool:
287
+ cfg = self.config
288
+ if not cfg.enable_batch:
289
+ return False
290
+ if not self._has_batch_config():
291
+ raise ValueError("enable_batch=True requires feature_batch_size or feature_batches.")
292
+ return True
293
+
275
294
  def _read_csv(self, csv_path: Path, usecols: list[str] | None = None, nrows: int | None = None) -> pd.DataFrame:
276
295
  if not csv_path.exists():
277
296
  raise FileNotFoundError(f"CSV file not found: {csv_path}")
@@ -325,6 +344,7 @@ class FeatureValidationPipeline:
325
344
  cfg,
326
345
  output_dir=str(batch_dir),
327
346
  input_type="dataframe",
347
+ enable_batch=False,
328
348
  feature_batch_size=None,
329
349
  feature_batches=None,
330
350
  new_feature_cols=list(batch_features),
@@ -250,6 +250,7 @@ _FIELD_LABELS = {
250
250
  "numeric_coercion_min_ratio": "安全数值转换阈值",
251
251
  "input_type": "输入类型",
252
252
  "csv_read_kwargs": "CSV 读取参数",
253
+ "enable_batch": "启用 CSV 分批",
253
254
  "feature_batch_size": "特征分批大小",
254
255
  "feature_batches": "显式特征批次",
255
256
  "batch_base_cols": "批处理基础列",
@@ -331,6 +332,7 @@ _FIELD_DESCRIPTIONS = {
331
332
  "online_data": "线上 DataFrame,仅代码模式可用。",
332
333
  "psi_reference_data": "外部 PSI benchmark DataFrame,仅代码模式可用。",
333
334
  "submodel_pairs": "子模型字段映射,GUI 可用 key=value 或 JSON 形式编辑。",
335
+ "enable_batch": "是否显式启用 CSV feature batch 模式;默认关闭。关闭时 feature_batch_size/feature_batches 仅保留在配置中,不会触发分批。",
334
336
  "feature_batch_size": "CSV 宽表模式下每批分析的新特征数量。",
335
337
  "feature_batches": "显式指定每批新特征列表;优先级高于 feature_batch_size。",
336
338
  "batch_corr_mode": "within_batch 只算批内相关性,block_pairwise 会额外读 CSV 计算跨批相关性。",
@@ -626,7 +628,7 @@ def _infer_group(name: str) -> str:
626
628
  return ANALYSIS_GROUP
627
629
  if "model" in name or name.startswith("lr_") or name.startswith("warm_start") or name.startswith("backward") or name.startswith("optuna") or name.startswith("explain") or name == "owen_enabled":
628
630
  return MODEL_GROUP
629
- if name in {"sql_dir", "offline_sql", "online_sql", "sqlrunner", "offline_data", "online_data", "input_type", "csv_read_kwargs"}:
631
+ if name in {"sql_dir", "offline_sql", "online_sql", "sqlrunner", "offline_data", "online_data", "input_type", "csv_read_kwargs", "enable_batch"}:
630
632
  return DATA_GROUP
631
633
  if name.startswith("perf") or name in {"nbins", "min_bin_prop", "equal_freq", "cross_vars", "cross_metrics"}:
632
634
  return EVAL_GROUP
@@ -687,7 +689,7 @@ def _field_meta(config_class: type, field_name: str, field_type: Any, default_va
687
689
  meta.depends_on = {"warm_start_enabled": True}
688
690
  if field_name.startswith("monotone_refine_") and field_name.endswith("_params"):
689
691
  meta.depends_on = {field_name.replace("_params", "_enabled"): True}
690
- if field_name in {"feature_batch_size", "feature_batches", "batch_corr_mode"}:
692
+ if field_name in {"enable_batch", "feature_batch_size", "feature_batches", "batch_corr_mode"}:
691
693
  meta.group = "CSV 分批"
692
694
  meta.advanced = True
693
695
  return meta
@@ -1028,8 +1030,13 @@ def validate_pipeline_config(pipeline_key: str, values: dict[str, Any] | Any) ->
1028
1030
  if int(vals.get("optuna_n_trials", 5) or 0) < 5:
1029
1031
  warnings.append("optuna_n_trials 建议至少为 5,过小的搜索轮数不稳定。")
1030
1032
  elif entry.key == "feature_validation":
1033
+ has_batch_config = bool(vals.get("feature_batches")) or vals.get("feature_batch_size") is not None
1031
1034
  if vals.get("feature_batch_size") is not None and int(vals["feature_batch_size"]) <= 0:
1032
1035
  errors.append("feature_batch_size 必须为正整数。")
1036
+ if vals.get("enable_batch") and not has_batch_config:
1037
+ errors.append("enable_batch=True 时必须配置 feature_batch_size 或 feature_batches。")
1038
+ if vals.get("enable_batch") is False and has_batch_config:
1039
+ warnings.append("enable_batch=False 时 feature_batch_size/feature_batches 不会触发 CSV 分批。")
1033
1040
  if vals.get("batch_corr_mode") == "block_pairwise":
1034
1041
  method = str((vals.get("corr_params") or {}).get("method", "pearson")).lower()
1035
1042
  if method == "kendall":
@@ -1,7 +1,7 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  __author__ = "Jingkai Sun"
4
- __version__ = "0.5.4"
4
+ __version__ = "0.5.5"
5
5
 
6
6
  from ._utils import SMF_MISSING_BIN
7
7
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SuperModelingFactory
3
- Version: 0.5.4
3
+ Version: 0.5.5
4
4
  Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
5
5
  Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
6
6
  Author: Kyle Sun
@@ -265,7 +265,7 @@ em.close_workbook()
265
265
 
266
266
  ## 版本
267
267
 
268
- - **Version**: 0.5.4
268
+ - **Version**: 0.5.5
269
269
  - **Author**: Jingkai Sun
270
270
 
271
271
  ## 许可证
@@ -208,7 +208,7 @@ em.close_workbook()
208
208
 
209
209
  ## 版本
210
210
 
211
- - **Version**: 0.5.4
211
+ - **Version**: 0.5.5
212
212
  - **Author**: Jingkai Sun
213
213
 
214
214
  ## 许可证
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SuperModelingFactory
3
- Version: 0.5.4
3
+ Version: 0.5.5
4
4
  Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
5
5
  Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
6
6
  Author: Kyle Sun
@@ -265,7 +265,7 @@ em.close_workbook()
265
265
 
266
266
  ## 版本
267
267
 
268
- - **Version**: 0.5.4
268
+ - **Version**: 0.5.5
269
269
  - **Author**: Jingkai Sun
270
270
 
271
271
  ## 许可证
@@ -12,7 +12,7 @@ build-backend = "setuptools.build_meta"
12
12
 
13
13
  [project]
14
14
  name = "SuperModelingFactory"
15
- version = "0.5.4"
15
+ version = "0.5.5"
16
16
  description = "Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting."
17
17
  readme = "README.md"
18
18
  requires-python = ">=3.10"
@@ -15,7 +15,7 @@ def _read(path: str) -> str:
15
15
 
16
16
  setup(
17
17
  name="SuperModelingFactory",
18
- version=os.environ.get("SMF_VERSION", "0.5.4"),
18
+ version=os.environ.get("SMF_VERSION", "0.5.5"),
19
19
  description="Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.",
20
20
  long_description=_read("README.md"),
21
21
  long_description_content_type="text/markdown",