SuperModelingFactory 0.5.5__tar.gz → 0.5.6__tar.gz

{supermodelingfactory-0.5.5 → supermodelingfactory-0.5.6}/Modeling_Tool/Core/ODPS_Tool.py

@@ -281,17 +281,31 @@ class ODPSRunner(object):
 
     @staticmethod
     def cre_table_schema(df, partition_name=None):
-
-        dtypes = df.dtypes
-        isnum_dtypes = [pd.api.types.is_numeric_dtype(x) for x in dtypes]
-        isint_dtypes = [pd.api.types.is_integer_dtype(x) for x in dtypes]
-
         table_columns = []
         table_partitions = []
-        for i in range(len(df.columns)):
-            col = df.columns[i]
-            col_type = "string" if not isnum_dtypes[i] else "float"
-            col_type = "bigint" if isint_dtypes[i] else col_type
+        for col, dtype in df.dtypes.items():
+            if pd.api.types.is_complex_dtype(dtype) or pd.api.types.is_timedelta64_dtype(dtype):
+                raise TypeError(
+                    f"Unsupported dtype for ODPS schema inference: column={col!r}, dtype={dtype!s}."
+                )
+            if pd.api.types.is_bool_dtype(dtype):
+                col_type = "boolean"
+            elif pd.api.types.is_integer_dtype(dtype):
+                col_type = "bigint"
+            elif pd.api.types.is_float_dtype(dtype):
+                col_type = "float" if str(dtype).lower() == "float32" else "double"
+            elif pd.api.types.is_datetime64_any_dtype(dtype):
+                col_type = "datetime"
+            elif (
+                pd.api.types.is_string_dtype(dtype)
+                or pd.api.types.is_object_dtype(dtype)
+                or isinstance(dtype, pd.CategoricalDtype)
+            ):
+                col_type = "string"
+            else:
+                raise TypeError(
+                    f"Unsupported dtype for ODPS schema inference: column={col!r}, dtype={dtype!s}."
+                )
             if col == partition_name:
                 table_partitions.append(Partition(name=col, type=col_type))
             else:
@@ -304,6 +318,28 @@ class ODPSRunner(object):
 
         return table_schema
 
+    @staticmethod
+    def _to_odps_records(df):
+        """Return Python records with pandas missing values converted to ODPS NULL."""
+        object_df = df.astype(object)
+        object_df = object_df.where(pd.notna(object_df), None)
+        return object_df.values.tolist()
+
+    def _execute_sql_blocking(self, sql):
+        """Execute DDL and wait until the remote ODPS instance has completed."""
+        execute_sql = getattr(self.o, "execute_sql", None)
+        if callable(execute_sql):
+            return execute_sql(sql)
+
+        run_sql = getattr(self.o, "run_sql", None)
+        if not callable(run_sql):
+            raise AttributeError("ODPS client must provide execute_sql() or run_sql().")
+        instance = run_sql(sql)
+        wait_for_success = getattr(instance, "wait_for_success", None)
+        if callable(wait_for_success):
+            wait_for_success()
+        return instance
+
     def upload_df(self, df, table_name, table_schema=None, partition=None, atomic=True):
         """上传数据集至mc中创建新表
 
@@ -328,9 +364,11 @@ class ODPSRunner(object):
             legacy behaviour, e.g. for tables whose downstream readers expect
             a specific object identity.
         """
+        upload_data = df.copy()
         if table_schema is None:
-            df.loc[:, "py_inserttime"] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-            table_schema = self.cre_table_schema(df=df, partition_name=None)
+            upload_data.loc[:, "py_inserttime"] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+            table_schema = self.cre_table_schema(df=upload_data, partition_name=None)
+        records = self._to_odps_records(upload_data)
 
         if not atomic:
             # Legacy pre-0.4.2 non-atomic path — retained under an explicit
@@ -339,10 +377,10 @@ class ODPSRunner(object):
             t = self.o.create_table(table_name, table_schema)
             if bool(partition):
                 with t.open_writer(partition=partition, create_partition=True) as writer:
-                    writer.write(df.values.tolist())
+                    writer.write(records)
             else:
                 with t.open_writer() as writer:
-                    writer.write(df.values.tolist())
+                    writer.write(records)
             logger.info(f'<<<< 完成数据入表{table_name}: shape={df.shape} >>>>')
             return
 
@@ -360,10 +398,10 @@ class ODPSRunner(object):
         try:
             if bool(partition):
                 with tmp_table.open_writer(partition=partition, create_partition=True) as writer:
-                    writer.write(df.values.tolist())
+                    writer.write(records)
             else:
                 with tmp_table.open_writer() as writer:
-                    writer.write(df.values.tolist())
+                    writer.write(records)
         except Exception:
             try:
                 self.o.delete_table(tmp_name, if_exists=True)
@@ -379,7 +417,7 @@ class ODPSRunner(object):
         target_exists = self.o.exist_table(table_name)
         if target_exists:
             try:
-                self.o.run_sql(f"ALTER TABLE {table_name} RENAME TO {old_rename_name};")
+                self._execute_sql_blocking(f"ALTER TABLE {table_name} RENAME TO {old_rename_name};")
             except Exception:
                 # Rename of the live target failed — do not touch it. Drop the
                 # tmp table so we don't leak it, then re-raise.
@@ -392,13 +430,13 @@ class ODPSRunner(object):
                 raise
 
         try:
-            self.o.run_sql(f"ALTER TABLE {tmp_name} RENAME TO {target_rename_name};")
+            self._execute_sql_blocking(f"ALTER TABLE {tmp_name} RENAME TO {target_rename_name};")
         except Exception:
             # tmp -> target rename failed. If we already moved the original,
             # try to restore it so the caller is not left with a missing table.
             if target_exists:
                 try:
-                    self.o.run_sql(f"ALTER TABLE {old_name} RENAME TO {target_rename_name};")
+                    self._execute_sql_blocking(f"ALTER TABLE {old_name} RENAME TO {target_rename_name};")
                     logger.warning(
                         f"upload_df: swap of {tmp_name} -> {table_name} failed; restored original from {old_name}"
                     )
@@ -439,7 +477,9 @@ class ODPSRunner(object):
     def _rename_partition(self, table_name, source_partition, target_partition):
         src = _format_odps_partition_spec(_parse_odps_partition_spec(source_partition), quoted=True)
         dst = _format_odps_partition_spec(_parse_odps_partition_spec(target_partition), quoted=True)
-        self.o.run_sql(f"ALTER TABLE {table_name} PARTITION ({src}) RENAME TO PARTITION ({dst});")
+        self._execute_sql_blocking(
+            f"ALTER TABLE {table_name} PARTITION ({src}) RENAME TO PARTITION ({dst});"
+        )
 
     def insert_df(self, df, table_name, overwrite=True, partition=None, atomic=True):
         """将数据集插入至mc已存在的表中.
@@ -457,7 +497,9 @@ class ODPSRunner(object):
         """
         t = self.o.get_table(table_name)
         if "py_inserttime" in t.schema and "py_inserttime" not in df.columns:
+            df = df.copy()
             df.loc[:, "py_inserttime"] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        records = self._to_odps_records(df)
 
         if bool(partition):
             if overwrite and atomic:
@@ -476,7 +518,7 @@ class ODPSRunner(object):
 
                 try:
                     with t.open_writer(partition=staging_partition, create_partition=True) as writer:
-                        writer.write(df.values.tolist())
+                        writer.write(records)
                     if target_exists:
                         self._rename_partition(table_name, partition, backup_partition)
                     self._rename_partition(table_name, staging_partition, partition)
@@ -510,10 +552,10 @@ class ODPSRunner(object):
             if overwrite:
                 t.delete_partition(partition, if_exists=True)
             with t.open_writer(partition=partition, create_partition=True) as writer:
-                writer.write(df.values.tolist())
+                writer.write(records)
         else:
             if overwrite:
                 t.truncate()
             with t.open_writer() as writer:
-                writer.write(df.values.tolist())
+                writer.write(records)
         logger.info('<<<< 完成数据入表: shape={0} >>>>'.format(df.shape))

{supermodelingfactory-0.5.5 → supermodelingfactory-0.5.6}/Modeling_Tool/Core/Proc_Compare.py

@@ -38,6 +38,7 @@ class ProcCompareConfig:
     numeric_tol: float = 1e-8
     numeric_rtol: float = 0.0
     datetime_tol_seconds: float = 0.0
+    datetime_cols: list[str] = field(default_factory=list)
     per_column_tolerance: dict[str, Any] = field(default_factory=dict)
 
     both_null_equal: bool = True
@@ -185,15 +186,27 @@ class ProcCompareEngine:
         cols = sorted((set(left_df.columns) | set(right_df.columns)) - {_ROW_ID_COL})
         rows = []
         ignore = set(self.config.ignore_cols)
-        compare_set = set(self._resolve_compare_cols(left_df, right_df, key_cols, allow_missing=True))
+        requested_compare_set = set(
+            self._resolve_compare_cols(left_df, right_df, key_cols, allow_missing=True)
+        )
         for col in cols:
             in_left = col in left_df.columns
             in_right = col in right_df.columns
+            requested_for_compare = col in requested_compare_set
+            eligible_for_compare = bool(
+                requested_for_compare
+                and in_left
+                and in_right
+                and col not in key_cols
+                and col not in ignore
+            )
             if col in key_cols:
                 role = "key"
+            elif not (in_left and in_right):
+                role = "schema_only"
             elif col in ignore:
                 role = "ignored"
-            elif col in compare_set:
+            elif eligible_for_compare:
                 role = "compare"
             else:
                 role = "not_compared"
@@ -207,6 +220,8 @@ class ProcCompareEngine:
                 {
                     "column": col,
                     "role": role,
+                    "requested_for_compare": bool(requested_for_compare),
+                    "eligible_for_compare": bool(eligible_for_compare),
                     "in_left": bool(in_left),
                     "in_right": bool(in_right),
                     "dtype_left": str(left_df[col].dtype) if in_left else "",
@@ -239,6 +254,7 @@ class ProcCompareEngine:
         right_df: pd.DataFrame,
         key_cols: list[str],
         schema_summary: pd.DataFrame | None = None,
+        keep_internal_stats: bool = False,
     ) -> ProcCompareResult:
         left_df, left_dups = self._handle_duplicates(left_df, key_cols, self.config.left_name)
         right_df, right_dups = self._handle_duplicates(right_df, key_cols, self.config.right_name)
@@ -349,7 +365,11 @@ class ProcCompareEngine:
         return ProcCompareResult(
             coverage_summary=coverage_summary,
             schema_summary=schema_summary,
-            column_summary=self._strip_internal_columns(column_summary),
+            column_summary=(
+                column_summary
+                if keep_internal_stats
+                else self._strip_internal_columns(column_summary)
+            ),
             row_summary=row_summary,
             cell_mismatches=cell_mismatches,
             duplicate_key_summary=duplicate_key_summary,
@@ -370,7 +390,7 @@ class ProcCompareEngine:
         diff = pd.Series(np.nan, index=left.index, dtype="float64")
         abs_diff = pd.Series(np.nan, index=left.index, dtype="float64")
 
-        if self._should_compare_datetime(left, right):
+        if self._should_compare_datetime(col, left, right):
             left_dt = pd.to_datetime(left, errors="coerce")
             right_dt = pd.to_datetime(right, errors="coerce")
             diff = (left_dt - right_dt).dt.total_seconds()
@@ -421,9 +441,28 @@ class ProcCompareEngine:
             return False
         return bool(left_num.notna().all() and right_num.notna().all())
 
-    @staticmethod
-    def _should_compare_datetime(left: pd.Series, right: pd.Series) -> bool:
-        return bool(pd.api.types.is_datetime64_any_dtype(left) or pd.api.types.is_datetime64_any_dtype(right))
+    def _should_compare_datetime(self, col: str, left: pd.Series, right: pd.Series) -> bool:
+        if pd.api.types.is_datetime64_any_dtype(left) or pd.api.types.is_datetime64_any_dtype(right):
+            return True
+        if col in self.config.datetime_cols:
+            return True
+
+        override = self.config.per_column_tolerance.get(col)
+        if isinstance(override, dict) and "datetime_tol_seconds" in override:
+            return True
+        if self.config.datetime_tol_seconds <= 0:
+            return False
+        if self._should_compare_numeric(left, right):
+            return False
+
+        values = pd.concat([left.dropna(), right.dropna()], ignore_index=True)
+        if values.empty:
+            return False
+        text = values.astype("string").str.strip()
+        if not text.str.match(r"^\d{4}[-/]\d{1,2}[-/]\d{1,2}(?:[ T].*)?$", na=False).all():
+            return False
+        parsed = pd.to_datetime(values, errors="coerce")
+        return bool(parsed.notna().all())
 
     def _column_numeric_tol(self, col: str) -> tuple[float, float]:
         override = self.config.per_column_tolerance.get(col)
@@ -547,7 +586,13 @@ class ProcCompareEngine:
         right_part_df = pd.read_csv(right_part) if right_part.exists() else pd.DataFrame(columns=right_columns)
         left_part_df = self._normalize_missing(left_part_df)
         right_part_df = self._normalize_missing(right_part_df)
-        return self._compare_frames(left_part_df, right_part_df, key_cols, schema_summary=schema_summary)
+        return self._compare_frames(
+            left_part_df,
+            right_part_df,
+            key_cols,
+            schema_summary=schema_summary,
+            keep_internal_stats=True,
+        )
 
     def _partition_input(self, data: pd.DataFrame | str | Path, out_dir: Path, key_cols: list[str]) -> None:
         if isinstance(data, pd.DataFrame):
@@ -589,21 +634,19 @@ class ProcCompareEngine:
             ]
         )
 
-        cols = pd.concat([p.column_summary.assign(_partial=idx) for idx, p in enumerate(partials)], ignore_index=True)
+        cols = pd.concat(
+            [p.column_summary.assign(_partial=idx) for idx, p in enumerate(partials)],
+            ignore_index=True,
+        )
         if len(cols):
-            cols["_diff_sum"] = pd.concat(
-                [
-                    p.column_summary.assign(_partial=idx).get("_diff_sum", pd.Series(dtype=float))
-                    for idx, p in enumerate(partials)
-                ],
-                ignore_index=True,
-            ) if "_diff_sum" in cols.columns else 0.0
             grouped = cols.groupby("column", dropna=False).agg(
                 n_compared=("n_compared", "sum"),
                 n_equal=("n_equal", "sum"),
                 n_mismatch=("n_mismatch", "sum"),
                 n_one_side_null=("n_one_side_null", "sum"),
                 n_both_null=("n_both_null", "sum"),
+                _diff_sum=("_diff_sum", "sum"),
+                _diff_count=("_diff_count", "sum"),
                 max_abs_diff=("max_abs_diff", "max"),
             ).reset_index()
             grouped["pct_mismatch"] = np.where(
@@ -611,7 +654,11 @@ class ProcCompareEngine:
                 grouped["n_mismatch"] / grouped["n_compared"] * 100,
                 0.0,
             )
-            grouped["mean_diff"] = np.nan
+            grouped["mean_diff"] = np.where(
+                grouped["_diff_count"] > 0,
+                grouped["_diff_sum"] / grouped["_diff_count"],
+                np.nan,
+            )
             column_summary = grouped[
                 [
                     "column",

{supermodelingfactory-0.5.5 → supermodelingfactory-0.5.6}/Modeling_Tool/Feature/Weighted_Screen.py

@@ -402,12 +402,14 @@ def _missing_rate_for_series(
             return 1.0
         valid = int(values.notna().sum())
         return 1.0 - float(valid) / float(total)
-    arr = values.to_numpy(dtype=float, copy=False)
-    finite = np.isfinite(arr) & values.notna().to_numpy()
+    present = values.notna().to_numpy()
+    if pd.api.types.is_numeric_dtype(values):
+        arr = values.to_numpy(dtype=float, copy=False)
+        present &= np.isfinite(arr)
     total_w = float(np.sum(w))
     if total_w <= 0:
         return 1.0
-    return 1.0 - float(np.sum(w[finite])) / total_w
+    return 1.0 - float(np.sum(w[present])) / total_w
 
 
 def _apply_missing_rate_stage(

{supermodelingfactory-0.5.5 → supermodelingfactory-0.5.6}/Modeling_Tool/Pipeline/_common.py

@@ -55,6 +55,63 @@ def as_list(value: Any | Iterable[Any] | None, default: list[Any] | None = None)
     return list(value)
 
 
+def normalize_group_specs(
+    value: Any,
+    *,
+    default_min_size: int | None = None,
+) -> list[dict[str, Any]]:
+    """Normalize supported Pipeline group-spec forms into named dictionaries."""
+    if value is None:
+        return []
+
+    if isinstance(value, Mapping):
+        raw_specs = []
+        for name, spec in value.items():
+            if isinstance(spec, Mapping):
+                item = dict(spec)
+                item.setdefault("name", str(name))
+            else:
+                item = {"name": str(name), "columns": as_list(spec)}
+            raw_specs.append(item)
+    else:
+        raw_specs = as_list(value)
+
+    normalized: list[dict[str, Any]] = []
+    seen_names: set[str] = set()
+    for idx, spec in enumerate(raw_specs):
+        if isinstance(spec, Mapping):
+            item = dict(spec)
+            columns = item.get("columns", item.get("cols"))
+            if columns is None:
+                raise ValueError(
+                    f"group_specs entry {idx} must contain 'columns'; got keys {sorted(item)}"
+                )
+            columns = as_list(columns)
+            name = str(item.get("name") or "_x_".join(str(col) for col in columns))
+            min_size = item.get("min_size", default_min_size)
+        else:
+            columns = as_list(spec)
+            name = "_x_".join(str(col) for col in columns) or f"group_{idx}"
+            min_size = default_min_size
+
+        if not columns or any(not isinstance(col, str) or not col.strip() for col in columns):
+            raise ValueError(
+                f"group_specs entry {idx} must define a non-empty list of column names; "
+                f"got {columns!r}"
+            )
+        if name in seen_names:
+            raise ValueError(f"group_specs contains duplicate name {name!r}")
+        seen_names.add(name)
+        normalized.append(
+            {
+                "name": name,
+                "columns": [str(col) for col in columns],
+                "min_size": default_min_size if min_size is None else int(min_size),
+            }
+        )
+    return normalized
+
+
 def merge_dict(base: Mapping[str, Any] | None, override: Mapping[str, Any] | None) -> dict[str, Any]:
     merged = dict(base or {})
     merged.update(dict(override or {}))

{supermodelingfactory-0.5.5 → supermodelingfactory-0.5.6}/Modeling_Tool/Pipeline/credit_model.py

@@ -929,6 +929,21 @@ class CreditModelPipeline:
         base_params = dict(cfg.model_params.get("lr", {}))
         standardize = bool(base_params.pop("standardize", False))
         lr = LRMaster(params=base_params or None, standardize=standardize)
+        allowed_search_params = {
+            "objective",
+            "primary_set",
+            "gap_ref_sets",
+            "metric",
+            "refit",
+            "verbose",
+        }
+        unknown_search_params = sorted(set(cfg.lr_search_params) - allowed_search_params)
+        if unknown_search_params:
+            raise ValueError(
+                f"Unsupported lr_search_params keys: {unknown_search_params}. "
+                f"Allowed keys are {sorted(allowed_search_params)}. "
+                "LRMaster.grid_search_params uses holdout eval_sets and does not accept cv."
+            )
         params = merge_dict(
             {
                 "objective": "oot_gap_penalized",

{supermodelingfactory-0.5.5 → supermodelingfactory-0.5.6}/Modeling_Tool/Pipeline/feature_validation.py

@@ -16,6 +16,7 @@ from ._common import (
     apply_woe_fit_query,
     as_list,
     make_dirs,
+    normalize_group_specs,
     safe_to_csv,
     split_oot_by_flag,
     validate_woe_fit_query_columns,
@@ -992,13 +993,10 @@ class FeatureValidationPipeline:
     def _group_specs(self, include_global: bool = True) -> dict[str, list[str]]:
         cfg = self.config
         if cfg.group_specs is not None:
-            if isinstance(cfg.group_specs, dict):
-                return {str(k): list(v) for k, v in cfg.group_specs.items()}
-            specs = {}
-            for idx, spec in enumerate(cfg.group_specs):
-                cols = [spec] if isinstance(spec, str) else list(spec)
-                specs["x".join(cols) or f"group_{idx}"] = cols
-            return specs
+            return {
+                str(spec["name"]): list(spec["columns"])
+                for spec in normalize_group_specs(cfg.group_specs)
+            }
         specs: dict[str, list[str]] = {}
         if include_global:
             specs["global"] = []

{supermodelingfactory-0.5.5 → supermodelingfactory-0.5.6}/Modeling_Tool/Pipeline/field_meta.py

@@ -567,9 +567,10 @@ def _build_pipeline_registry() -> dict[str, PipelineRegistryEntry]:
             result_attrs=[
                 "label_coverage_summary",
                 "segment_bad_rate_summary",
+                "profile_summary",
                 "split_candidate_summary",
                 "split_recommendation",
-                "report_path",
+                "output_paths",
             ],
         ),
         "mock_sample": PipelineRegistryEntry(
@@ -1029,6 +1030,17 @@ def validate_pipeline_config(pipeline_key: str, values: dict[str, Any] | Any) ->
             errors.append("optuna_n_trials 必须 >= 1。")
         if int(vals.get("optuna_n_trials", 5) or 0) < 5:
             warnings.append("optuna_n_trials 建议至少为 5，过小的搜索轮数不稳定。")
+        allowed_lr_search_params = {
+            "objective", "primary_set", "gap_ref_sets", "metric", "refit", "verbose"
+        }
+        unknown_lr_search_params = sorted(
+            set(vals.get("lr_search_params") or {}) - allowed_lr_search_params
+        )
+        if unknown_lr_search_params:
+            errors.append(
+                f"Unsupported lr_search_params keys: {unknown_lr_search_params}; "
+                f"allowed keys are {sorted(allowed_lr_search_params)}."
+            )
     elif entry.key == "feature_validation":
         has_batch_config = bool(vals.get("feature_batches")) or vals.get("feature_batch_size") is not None
         if vals.get("feature_batch_size") is not None and int(vals["feature_batch_size"]) <= 0:
@@ -1057,6 +1069,21 @@ def validate_pipeline_config(pipeline_key: str, values: dict[str, Any] | Any) ->
             errors.append("target_col 不能为空。")
         if missing("score_cols") and missing("base_score"):
             warnings.append("未配置 score_cols/base_score 时将依赖 Pipeline 自动探测分数字段。")
+        if vals.get("group_specs") is not None:
+            try:
+                from ._common import normalize_group_specs
+
+                normalize_group_specs(vals["group_specs"])
+            except (TypeError, ValueError) as exc:
+                errors.append(f"Invalid group_specs: {exc}")
+        for name, spec in (vals.get("cross_metrics") or {}).items():
+            if not isinstance(spec, (list, tuple)) or len(spec) != 2:
+                errors.append(
+                    f"cross_metrics[{name!r}] must be a two-item [column, aggregation] pair."
+                )
+        pairwise_agg = vals.get("pairwise_cross_agg_dict")
+        if pairwise_agg is not None and not isinstance(pairwise_agg, dict):
+            errors.append("pairwise_cross_agg_dict must be a {column: aggregation(s)} mapping.")
     elif entry.key == "score_consistency_uat":
         if missing("main_model_score_col"):
             errors.append("main_model_score_col 不能为空。")