mflowy 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. mflowy/__init__.py +1 -0
  2. mflowy/compute/cleaners/filter/common_filter.py +55 -0
  3. mflowy/compute/cleaners/filter/correlation_filter.py +153 -0
  4. mflowy/compute/cleaners/filter/variance_filter.py +129 -0
  5. mflowy/compute/cleaners/missing/drop_handler.py +51 -0
  6. mflowy/compute/cleaners/missing/fill_handler.py +121 -0
  7. mflowy/compute/cleaners/outlier/iqr_detector.py +106 -0
  8. mflowy/compute/cleaners/outlier/zscore_detector.py +111 -0
  9. mflowy/compute/cleaners/python_cleaner.py +64 -0
  10. mflowy/compute/cleaners/unit/strip_units.py +49 -0
  11. mflowy/compute/cross_validation/group_k_fold.py +60 -0
  12. mflowy/compute/cross_validation/k_fold.py +43 -0
  13. mflowy/compute/cross_validation/leave_one_group_out.py +52 -0
  14. mflowy/compute/cross_validation/leave_one_out.py +33 -0
  15. mflowy/compute/cross_validation/repeated_k_fold.py +49 -0
  16. mflowy/compute/cross_validation/repeated_stratified_k_fold.py +45 -0
  17. mflowy/compute/cross_validation/simple_cv.py +73 -0
  18. mflowy/compute/cross_validation/stratified_group_k_fold.py +55 -0
  19. mflowy/compute/cross_validation/stratified_k_fold.py +54 -0
  20. mflowy/compute/cross_validation/types.py +11 -0
  21. mflowy/compute/loaders/__init__.py +8 -0
  22. mflowy/compute/loaders/csv_loader.py +31 -0
  23. mflowy/compute/loaders/excel_loader.py +31 -0
  24. mflowy/compute/loaders/file_loader.py +48 -0
  25. mflowy/compute/loaders/http_loader.py +179 -0
  26. mflowy/compute/loaders/parquet_loader.py +30 -0
  27. mflowy/compute/loaders/python_loader.py +64 -0
  28. mflowy/compute/model/_catboost.py +120 -0
  29. mflowy/compute/model/_embedded_tree.py +28 -0
  30. mflowy/compute/model/_evaluation_plots.py +77 -0
  31. mflowy/compute/model/_feature_importance.py +12 -0
  32. mflowy/compute/model/_lightgbm.py +103 -0
  33. mflowy/compute/model/_loss_curve.py +76 -0
  34. mflowy/compute/model/_mlp.py +56 -0
  35. mflowy/compute/model/_model.py +60 -0
  36. mflowy/compute/model/_names.py +22 -0
  37. mflowy/compute/model/_neural_network.py +488 -0
  38. mflowy/compute/model/_pipeline.py +395 -0
  39. mflowy/compute/model/_random_forest.py +68 -0
  40. mflowy/compute/model/_x_processors.py +16 -0
  41. mflowy/compute/model/_xgboost.py +81 -0
  42. mflowy/compute/model/catboost.py +92 -0
  43. mflowy/compute/model/lightgbm.py +103 -0
  44. mflowy/compute/model/loader.py +89 -0
  45. mflowy/compute/model/mlp.py +76 -0
  46. mflowy/compute/model/predict.py +94 -0
  47. mflowy/compute/model/random_forest.py +87 -0
  48. mflowy/compute/model/search_input.py +196 -0
  49. mflowy/compute/model/types.py +341 -0
  50. mflowy/compute/model/utils.py +48 -0
  51. mflowy/compute/model/xgboost.py +104 -0
  52. mflowy/compute/plots/base.py +115 -0
  53. mflowy/compute/plots/data_analysis/_grid.py +57 -0
  54. mflowy/compute/plots/data_analysis/correlation_heatmap.py +279 -0
  55. mflowy/compute/plots/data_analysis/numeric_quality_kde_hist.py +102 -0
  56. mflowy/compute/plots/data_analysis/numeric_scale_box.py +142 -0
  57. mflowy/compute/plots/data_analysis/target_association_by_category.py +82 -0
  58. mflowy/compute/plots/data_analysis/target_effect_by_category.py +91 -0
  59. mflowy/compute/plots/data_analysis/target_separation_by_numeric.py +88 -0
  60. mflowy/compute/plots/data_analysis/target_trend_by_numeric.py +177 -0
  61. mflowy/compute/plots/generation_quality/__init__.py +0 -0
  62. mflowy/compute/plots/generation_quality/_utils.py +35 -0
  63. mflowy/compute/plots/generation_quality/correlation_comparison.py +134 -0
  64. mflowy/compute/plots/generation_quality/cumulative_sums.py +132 -0
  65. mflowy/compute/plots/generation_quality/distribution_overlay.py +186 -0
  66. mflowy/compute/plots/generation_quality/mean_std_scatter.py +143 -0
  67. mflowy/compute/plots/model_evaluation/classification/confusion_matrix.py +111 -0
  68. mflowy/compute/plots/model_evaluation/loss_curve.py +85 -0
  69. mflowy/compute/plots/model_evaluation/regression/_taylor_diagram_copin.py +259 -0
  70. mflowy/compute/plots/model_evaluation/regression/error_distribution.py +136 -0
  71. mflowy/compute/plots/model_evaluation/regression/prediction_scatter.py +208 -0
  72. mflowy/compute/plots/model_evaluation/regression/residual_scatter.py +122 -0
  73. mflowy/compute/plots/model_evaluation/regression/taylor_diagram.py +196 -0
  74. mflowy/compute/plots/model_interpretability/shap/dependence.py +282 -0
  75. mflowy/compute/plots/model_interpretability/shap/sample_waterfall.py +60 -0
  76. mflowy/compute/plots/model_interpretability/shap/summary.py +152 -0
  77. mflowy/compute/plots/utils.py +185 -0
  78. mflowy/compute/statistic/__init__.py +0 -0
  79. mflowy/compute/statistic/effect_size.py +123 -0
  80. mflowy/compute/statistic/profile.py +89 -0
  81. mflowy/compute/synthesis/constraint/__init__.py +23 -0
  82. mflowy/compute/synthesis/constraint/base.py +27 -0
  83. mflowy/compute/synthesis/constraint/enum.py +55 -0
  84. mflowy/compute/synthesis/constraint/fixed_combination.py +44 -0
  85. mflowy/compute/synthesis/constraint/minmax.py +32 -0
  86. mflowy/compute/synthesis/constraint/negative.py +28 -0
  87. mflowy/compute/synthesis/constraint/normalize_to_one.py +40 -0
  88. mflowy/compute/synthesis/constraint/pipeline.py +99 -0
  89. mflowy/compute/synthesis/constraint/positive.py +28 -0
  90. mflowy/compute/synthesis/constraint/range.py +37 -0
  91. mflowy/compute/synthesis/constraints.py +164 -0
  92. mflowy/compute/synthesis/ctgan/__init__.py +79 -0
  93. mflowy/compute/synthesis/ctgan/activation.py +25 -0
  94. mflowy/compute/synthesis/ctgan/discriminator.py +28 -0
  95. mflowy/compute/synthesis/ctgan/generator.py +28 -0
  96. mflowy/compute/synthesis/ctgan/penalty.py +23 -0
  97. mflowy/compute/synthesis/ctgan/synthesizer.py +216 -0
  98. mflowy/compute/synthesis/ctgan/transformer.py +92 -0
  99. mflowy/compute/synthesis/infer.py +74 -0
  100. mflowy/compute/x_transformer/categorical/hash_encoder.py +49 -0
  101. mflowy/compute/x_transformer/categorical/label_encoder.py +61 -0
  102. mflowy/compute/x_transformer/categorical/onehot_encoder.py +38 -0
  103. mflowy/compute/x_transformer/categorical/ordinal_encoder.py +36 -0
  104. mflowy/compute/x_transformer/categorical/target_encoder.py +52 -0
  105. mflowy/compute/x_transformer/numeric/interaction_creator.py +115 -0
  106. mflowy/compute/x_transformer/numeric/log_transformer.py +86 -0
  107. mflowy/compute/x_transformer/numeric/minmax_scaler.py +32 -0
  108. mflowy/compute/x_transformer/numeric/numerical_binner.py +104 -0
  109. mflowy/compute/x_transformer/numeric/pca_reducer.py +33 -0
  110. mflowy/compute/x_transformer/numeric/power_transformer.py +38 -0
  111. mflowy/compute/x_transformer/numeric/robust_scaler.py +37 -0
  112. mflowy/compute/x_transformer/numeric/standard_scaler.py +29 -0
  113. mflowy/compute/x_transformer/utils.py +37 -0
  114. mflowy/compute/x_y.py +74 -0
  115. mflowy/driver/builder.py +227 -0
  116. mflowy/driver/builder_options.py +135 -0
  117. mflowy/driver/config.py +154 -0
  118. mflowy/driver/context.py +90 -0
  119. mflowy/driver/discover.py +90 -0
  120. mflowy/driver/handler.py +133 -0
  121. mflowy/driver/module.py +153 -0
  122. mflowy/driver/serializer.py +54 -0
  123. mflowy/driver/workflow.py +231 -0
  124. mflowy/mcp/__init__.py +3 -0
  125. mflowy/mcp/job_provider/__init__.py +57 -0
  126. mflowy/mcp/job_provider/local.py +244 -0
  127. mflowy/mcp/job_provider/protocol.py +58 -0
  128. mflowy/mcp/runner.py +57 -0
  129. mflowy/mcp/server.py +64 -0
  130. mflowy/mcp/telemetry.py +308 -0
  131. mflowy/mcp/templates/data_profile.yaml.j2 +28 -0
  132. mflowy/mcp/templates/eda.yaml.j2 +67 -0
  133. mflowy/mcp/templates/explanation.yaml.j2 +19 -0
  134. mflowy/mcp/templates/infer_task_type.yaml.j2 +20 -0
  135. mflowy/mcp/templates/inverse_optimization.yaml.j2 +32 -0
  136. mflowy/mcp/templates/modeling.yaml.j2 +19 -0
  137. mflowy/mcp/templates/predict.yaml.j2 +15 -0
  138. mflowy/mcp/tools.py +392 -0
  139. mflowy/middlewares/__init__.py +22 -0
  140. mflowy/middlewares/data_inject.py +238 -0
  141. mflowy/middlewares/inject_df_or_none.py +17 -0
  142. mflowy/middlewares/log_X_y.py +16 -0
  143. mflowy/middlewares/log_cv.py +34 -0
  144. mflowy/middlewares/log_df_diff.py +39 -0
  145. mflowy/middlewares/log_load_profile.py +35 -0
  146. mflowy/middlewares/log_plot.py +87 -0
  147. mflowy/middlewares/log_prediction.py +17 -0
  148. mflowy/middlewares/log_search_input.py +19 -0
  149. mflowy/middlewares/log_statistic.py +25 -0
  150. mflowy/middlewares/log_synthetic_df.py +16 -0
  151. mflowy/middlewares/mlflow_log.py +77 -0
  152. mflowy/middlewares/stop_on_error.py +29 -0
  153. mflowy/utils/capture.py +69 -0
  154. mflowy/utils/constants.py +4 -0
  155. mflowy/utils/df_columns.py +123 -0
  156. mflowy/utils/file.py +18 -0
  157. mflowy/utils/jinja.py +72 -0
  158. mflowy/utils/logging.py +112 -0
  159. mflowy/utils/mlflow.py +164 -0
  160. mflowy/utils/path.py +58 -0
  161. mflowy/utils/python_script_security_scan.py +173 -0
  162. mflowy/utils/study.py +104 -0
  163. mflowy/utils/utils.py +5 -0
  164. mflowy/utils/wraps.py +94 -0
  165. mflowy-0.1.0.dist-info/METADATA +176 -0
  166. mflowy-0.1.0.dist-info/RECORD +169 -0
  167. mflowy-0.1.0.dist-info/WHEEL +4 -0
  168. mflowy-0.1.0.dist-info/entry_points.txt +3 -0
  169. mflowy-0.1.0.dist-info/licenses/LICENSE +21 -0
mflowy/__init__.py ADDED
@@ -0,0 +1 @@
1
+ """MFlowy — MCP-native modular ML workflow engine."""
@@ -0,0 +1,55 @@
1
+ """列过滤器 - 实体实现
2
+
3
+ 功能:按列名、列值过滤数据
4
+ """
5
+
6
+ import logging
7
+ from typing import Annotated
8
+
9
+ import pandas as pd
10
+
11
+ from mflowy.driver.config import StepType
12
+ from mflowy.driver.handler import handler
13
+ from mflowy.middlewares.data_inject import inject_df
14
+ from mflowy.middlewares.log_df_diff import df_diff
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+
19
+ @handler(StepType.CLEAN, inject_df, df_diff)
20
+ def common_filter(
21
+ df: pd.DataFrame,
22
+ *,
23
+ drop: Annotated[list[str] | None, "排除指定列"] = None,
24
+ remain: Annotated[list[str] | None, "仅保留指定列"] = None,
25
+ **kwargs,
26
+ ) -> pd.DataFrame:
27
+ """按列名做"丢弃"或"保留"两种集合操作之一(仅作用于列,不作用于行)。
28
+
29
+ drop:从结果中删除指定列;remain:仅保留 remain 中且仍存在于 df.columns 的列、其余删除。两者可同时使用:drop 先执行,remain 再在剩余列上执行。drop 与 remain 中不存在于 df.columns 的列名被静默忽略;remain 中缺失的列名额外打印 warning。两者都为 None 时原样返回。
30
+
31
+ 用于"硬性按业务规则裁列"场景(如去掉 ID 列、只留特征列),不涉及任何数值统计。
32
+
33
+ common_filter 用"按列名手工选/删列"场景,variance_filter / correlation_filter 用"按统计指标自动筛列"场景。
34
+ """
35
+ cleaned_df = df
36
+
37
+ if drop:
38
+ drop_cols = [col for col in drop if col in cleaned_df.columns]
39
+ if drop_cols:
40
+ cleaned_df = cleaned_df.drop(columns=drop_cols)
41
+ logger.debug(f"Dropped {','.join(drop_cols)}")
42
+
43
+ if remain:
44
+ keep_cols = [col for col in remain if col in cleaned_df.columns]
45
+ missing_cols = set(remain) - set(keep_cols)
46
+ drop_cols = [col for col in cleaned_df.columns if col not in keep_cols]
47
+
48
+ if missing_cols:
49
+ logger.warning(f"{len(missing_cols)} columns not found: {missing_cols}")
50
+
51
+ if drop_cols:
52
+ cleaned_df = cleaned_df.drop(columns=drop_cols)
53
+ logger.debug(f"Dropped {','.join(drop_cols)}")
54
+
55
+ return cleaned_df
@@ -0,0 +1,153 @@
1
+ """相关性过滤器 - 实体实现
2
+
3
+ 功能:基于特征相关性阈值过滤冗余特征
4
+ """
5
+
6
+ import logging
7
+ from itertools import combinations
8
+ from typing import Annotated, Literal
9
+
10
+ import pandas as pd
11
+
12
+ from mflowy.driver.config import StepType
13
+ from mflowy.driver.handler import handler
14
+ from mflowy.middlewares.data_inject import inject_df
15
+ from mflowy.middlewares.log_df_diff import df_diff
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+
20
+ @handler(StepType.CLEAN, inject_df, df_diff)
21
+ def correlation_filter(
22
+ df: pd.DataFrame,
23
+ *,
24
+ threshold: Annotated[float, "相关系数阈值 (0~1),超过则移除其中一个特征"] = 0.9,
25
+ method: Annotated[Literal["pearson", "spearman", "kendall"], "相关性计算方法"] = "spearman",
26
+ priority_features: Annotated[list[str] | None, "优先保留的特征列表"] = None,
27
+ exclude: Annotated[list[str] | None, "不参与计算和过滤的列(如目标列)"] = None,
28
+ **kwargs,
29
+ ) -> pd.DataFrame:
30
+ """基于相关性阈值过滤高度共线的数值特征(仅删列)。
31
+
32
+ 仅对数值列计算相关系数矩阵(method 默认 spearman,可切 pearson/kendall),|r| ≥ threshold(默认 0.9)的特征对视为冗余。两两比较后保留 priority_features 中优先项,否则保留方差更大者;exclude 列不参与计算且不被删除。数值列 <2 时原样返回。
33
+
34
+ 用于"线性模型/可解释模型去除共线特征"场景。threshold∈[0,1]、method 非法均抛 ValueError。
35
+
36
+ correlation_filter 用"剔除冗余共线特征"场景,variance_filter 用"剔除低信息量特征"场景,iqr/zscore_detector 用"剔除异常样本(按行)"场景。
37
+ """
38
+ # 参数验证
39
+ if not 0 <= threshold <= 1:
40
+ raise ValueError(f"threshold must be between 0 and 1, got {threshold}")
41
+
42
+ if method not in ["pearson", "spearman", "kendall"]:
43
+ raise ValueError(f"method must be 'pearson', 'spearman', or 'kendall', got {method}")
44
+
45
+ # 记录原始形状
46
+ original_rows, original_cols = df.shape
47
+
48
+ # 识别数值列,并排除指定列
49
+ exclude_set = set(exclude) if exclude else set()
50
+ all_numeric_cols = df.select_dtypes(include=["number"]).columns.tolist()
51
+ numeric_cols = [col for col in all_numeric_cols if col not in exclude_set]
52
+ excluded_cols = [col for col in all_numeric_cols if col in exclude_set]
53
+
54
+ if len(numeric_cols) < 2:
55
+ logger.debug(
56
+ f"Less than 2 numeric columns found for correlation analysis ({len(numeric_cols)}), returning original data"
57
+ )
58
+ return df.copy()
59
+
60
+ # 计算相关性矩阵
61
+ corr_matrix = df[numeric_cols].corr(method=method)
62
+
63
+ # 识别高相关特征对
64
+ correlated_pairs = _find_correlated_pairs(corr_matrix, threshold=threshold)
65
+
66
+ if not correlated_pairs:
67
+ logger.debug(f"No highly correlated pairs found (threshold={threshold}, method={method})")
68
+ return df.copy()
69
+
70
+ # 确定要移除的特征
71
+ features_to_remove = _select_features_to_remove(
72
+ correlated_pairs=correlated_pairs,
73
+ data=df,
74
+ priority_features=priority_features or [],
75
+ )
76
+
77
+ # 执行过滤
78
+ non_numeric_cols = [col for col in df.columns if col not in all_numeric_cols]
79
+ final_cols = non_numeric_cols + excluded_cols + [col for col in numeric_cols if col not in features_to_remove]
80
+
81
+ result = df[final_cols]
82
+
83
+ logger.debug(
84
+ f"CorrelationFilter (threshold={threshold}, method={method}): {original_rows}x{original_cols} -> {result.shape[0]}x{result.shape[1]}, removed {len(features_to_remove)} features"
85
+ )
86
+
87
+ return result
88
+
89
+
90
+ # ---------------------------------------------------------------------------
91
+ # 内部辅助函数
92
+ # ---------------------------------------------------------------------------
93
+
94
+
95
+ def _find_correlated_pairs(
96
+ corr_matrix: pd.DataFrame,
97
+ threshold: float,
98
+ ) -> list[tuple[str, str, float]]:
99
+ """识别高相关特征对"""
100
+ correlated_pairs = []
101
+
102
+ for feat1, feat2 in combinations(corr_matrix.columns, 2):
103
+ corr_value = corr_matrix.loc[feat1, feat2]
104
+
105
+ if abs(corr_value) >= threshold:
106
+ correlated_pairs.append((feat1, feat2, abs(corr_value)))
107
+
108
+ correlated_pairs.sort(key=lambda x: x[2], reverse=True)
109
+
110
+ return correlated_pairs
111
+
112
+
113
+ def _select_features_to_remove(
114
+ correlated_pairs: list[tuple[str, str, float]],
115
+ data: pd.DataFrame,
116
+ priority_features: list[str],
117
+ ) -> list[str]:
118
+ """从高相关特征对中选择要移除的特征
119
+
120
+ 策略:
121
+ 1. 优先保留 priority_features 中的特征
122
+ 2. 在其余特征中,保留方差更大的特征
123
+ """
124
+ features_to_remove = []
125
+ removed_features = set()
126
+
127
+ numeric_cols = data.select_dtypes(include=["number"]).columns
128
+ feature_variances = data[numeric_cols].var().to_dict()
129
+
130
+ priority_set = set(priority_features)
131
+
132
+ for feat1, feat2, corr_value in correlated_pairs:
133
+ if feat1 in removed_features or feat2 in removed_features:
134
+ continue
135
+
136
+ if feat1 in priority_set and feat2 not in priority_set:
137
+ keep_feat, remove_feat = feat1, feat2
138
+ elif feat2 in priority_set and feat1 not in priority_set:
139
+ keep_feat, remove_feat = feat2, feat1
140
+ else:
141
+ if feature_variances.get(feat1, 0) >= feature_variances.get(feat2, 0):
142
+ keep_feat, remove_feat = feat1, feat2
143
+ else:
144
+ keep_feat, remove_feat = feat2, feat1
145
+
146
+ features_to_remove.append(remove_feat)
147
+ removed_features.add(remove_feat)
148
+
149
+ logger.debug(
150
+ f"High correlation detected: {feat1} <-> {feat2} (r={corr_value:.3f}), removing '{remove_feat}', keeping '{keep_feat}' (variance: {feature_variances.get(remove_feat, 0):.4f} vs {feature_variances.get(keep_feat, 0):.4f})"
151
+ )
152
+
153
+ return features_to_remove
@@ -0,0 +1,129 @@
1
+ """方差过滤器 - 实体实现
2
+
3
+ 功能:基于方差阈值过滤列或行
4
+ """
5
+
6
+ import logging
7
+ from typing import Annotated
8
+
9
+ import pandas as pd
10
+
11
+ from mflowy.driver.config import StepType
12
+ from mflowy.driver.handler import handler
13
+ from mflowy.middlewares.data_inject import inject_df
14
+ from mflowy.middlewares.log_df_diff import df_diff
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+
19
+ @handler(StepType.CLEAN, inject_df, df_diff)
20
+ def variance_filter(
21
+ df: pd.DataFrame,
22
+ *,
23
+ axis: Annotated[int, "过滤方向 (0=按列, 1=按行)"] = 0,
24
+ threshold: Annotated[float, "方差阈值,低于此值将被过滤"] = 0.0,
25
+ column_thresholds: Annotated[dict[str, float] | None, "按列覆盖的特定阈值"] = None,
26
+ **kwargs,
27
+ ) -> pd.DataFrame:
28
+ """按方差阈值过滤列(axis=0)或行(axis=1)。
29
+
30
+ 仅作用于数值列;axis=0 时删除数值列方差 < threshold 的列(保留非数值列),axis=1 时删除数值行方差 < threshold 的行。threshold 默认 0.0(即删除常量列/常量行)。column_thresholds 可按列名/行索引覆盖单一阈值。
31
+
32
+ 用于"剔除常量或近常量特征/样本"场景,如方差=0 的列无法区分样本。负数阈值抛 ValueError。
33
+
34
+ variance_filter 用"按方差筛掉低信息列"场景,correlation_filter 用"按相关性筛掉冗余列"场景,common_filter 用"按业务规则硬选/硬删列名"场景。
35
+ """
36
+ # 参数验证
37
+ if axis not in [0, 1]:
38
+ raise ValueError(f"axis must be 0 or 1, got {axis}")
39
+
40
+ if threshold < 0:
41
+ raise ValueError(f"threshold must be non-negative, got {threshold}")
42
+
43
+ if column_thresholds:
44
+ for key, value in column_thresholds.items():
45
+ if value < 0:
46
+ raise ValueError(f"column_thresholds['{key}'] must be non-negative, got {value}")
47
+
48
+ # 记录原始形状
49
+ original_rows, original_cols = df.shape
50
+
51
+ # 执行过滤
52
+ if axis == 0:
53
+ result = _filter_columns(df, threshold=threshold, column_thresholds=column_thresholds)
54
+ else:
55
+ result = _filter_rows(df, threshold=threshold, column_thresholds=column_thresholds)
56
+
57
+ logger.debug(
58
+ f"VarianceFilter (axis={axis}): {original_rows}x{original_cols} -> {result.shape[0]}x{result.shape[1]}"
59
+ )
60
+
61
+ return result
62
+
63
+
64
+ # ---------------------------------------------------------------------------
65
+ # 内部辅助函数
66
+ # ---------------------------------------------------------------------------
67
+
68
+
69
+ def _filter_columns(
70
+ data: pd.DataFrame,
71
+ *,
72
+ threshold: float,
73
+ column_thresholds: dict[str, float] | None,
74
+ ) -> pd.DataFrame:
75
+ """过滤低方差列"""
76
+ df = data.copy()
77
+
78
+ numeric_cols = df.select_dtypes(include=["number"]).columns.tolist()
79
+ if not numeric_cols:
80
+ logger.debug("No numeric columns found, returning original data")
81
+ return df
82
+
83
+ variances = df[numeric_cols].var()
84
+
85
+ columns_to_keep = []
86
+ columns_to_remove = []
87
+
88
+ for col in numeric_cols:
89
+ col_threshold = column_thresholds.get(col, threshold) if column_thresholds else threshold
90
+
91
+ if variances[col] >= col_threshold:
92
+ columns_to_keep.append(col)
93
+ else:
94
+ columns_to_remove.append(col)
95
+ logger.debug(f"Column '{col}' removed: variance={variances[col]:.4f} < threshold={col_threshold:.4f}")
96
+
97
+ non_numeric_cols = [col for col in df.columns if col not in numeric_cols]
98
+ final_cols = non_numeric_cols + columns_to_keep
99
+
100
+ return df[final_cols]
101
+
102
+
103
+ def _filter_rows(
104
+ data: pd.DataFrame,
105
+ *,
106
+ threshold: float,
107
+ column_thresholds: dict[str, float] | None,
108
+ ) -> pd.DataFrame:
109
+ """过滤低方差行"""
110
+ df = data.copy()
111
+
112
+ numeric_cols = df.select_dtypes(include=["number"]).columns.tolist()
113
+ if not numeric_cols:
114
+ logger.debug("No numeric columns found, returning original data")
115
+ return df
116
+
117
+ row_variances = df[numeric_cols].var(axis=1)
118
+
119
+ rows_to_keep = []
120
+
121
+ for idx in df.index:
122
+ row_threshold = column_thresholds.get(str(idx), threshold) if column_thresholds else threshold
123
+
124
+ if row_variances[idx] >= row_threshold:
125
+ rows_to_keep.append(idx)
126
+ else:
127
+ logger.debug(f"Row {idx} removed: variance={row_variances[idx]:.4f} < threshold={row_threshold:.4f}")
128
+
129
+ return df.loc[rows_to_keep]
@@ -0,0 +1,51 @@
1
+ """删除缺失值处理器"""
2
+
3
+ import logging
4
+ from typing import Annotated
5
+
6
+ import pandas as pd
7
+
8
+ from mflowy.driver.config import StepType
9
+ from mflowy.driver.handler import handler
10
+ from mflowy.middlewares.data_inject import inject_df
11
+ from mflowy.middlewares.log_df_diff import df_diff
12
+
13
+ logger = logging.getLogger(__name__)
14
+
15
+
16
+ @handler(StepType.CLEAN, inject_df, df_diff)
17
+ def drop_missing(
18
+ df: pd.DataFrame,
19
+ *,
20
+ threshold: Annotated[float, "[0, 1.0],全局缺失率门禁, ≥ threshold 的列将被删除"] = 0.6,
21
+ column_threshold: Annotated[dict[str, float] | None, "为不同列配置不同的缺失率门禁,不配置时,采用全局门禁"] = None,
22
+ exclude: Annotated[list[str] | None, "不做缺失值门禁检查的列"] = None,
23
+ **kwargs,
24
+ ) -> pd.DataFrame:
25
+ """按列删除缺失率超阈值的列(按列删除,非按行)。
26
+
27
+ 仅遍历列、比较 isnull().mean() ≥ 阈值即删列,不删行。threshold 为 [0,1] 全局门禁(默认 0.6);column_threshold 为单列覆盖;阈值=1 或 <0 时直接返回原表(no-op)。
28
+
29
+ 用于"剔除数据质量过差的列"场景(如缺失率 ≥60% 的列),不适用于"删除任意含缺失的行"。column_threshold 用于差异化门禁场景(如关键业务列放宽到 0.9、辅助列收紧到 0.3)。
30
+
31
+ drop_missing 用"整列丢弃"场景,fill_missing 用"保留行但插补"场景。
32
+ """
33
+ column_threshold = column_threshold or {}
34
+ threshold = min(threshold, 1)
35
+ if threshold < 0:
36
+ return df
37
+ if threshold == 1:
38
+ return df
39
+
40
+ cleaned_df = df
41
+ missing_rates = cleaned_df.isnull().mean()
42
+
43
+ cols_to_drop = []
44
+ for col, missing_rate in missing_rates.items():
45
+ gate = column_threshold.get(str(col), threshold)
46
+ if missing_rate >= gate:
47
+ cols_to_drop.append(col)
48
+ logger.debug(f"{col} with missing rate >= {gate} will be Dropped")
49
+ if cols_to_drop:
50
+ cleaned_df = cleaned_df.drop(columns=cols_to_drop)
51
+ return cleaned_df
@@ -0,0 +1,121 @@
1
+ """填充缺失值处理器"""
2
+
3
+ import logging
4
+ from typing import Annotated, Literal
5
+
6
+ import pandas as pd
7
+
8
+ from mflowy.driver.config import StepType
9
+ from mflowy.driver.handler import handler
10
+ from mflowy.middlewares.data_inject import inject_df
11
+ from mflowy.middlewares.log_df_diff import df_diff
12
+
13
+ logger = logging.getLogger(__name__)
14
+
15
+ type _MISSING_STRATEGY = Literal[
16
+ "mean", "median", "interpolate", "forward", "backward", "mode", "fill", "fill_grouped_mode"
17
+ ]
18
+
19
+ EMPTY_GROUPY_BY = ValueError("缺失值填充策略 fill_grouped_mode 的分组列为空")
20
+
21
+
22
+ @handler(StepType.CLEAN, inject_df, df_diff)
23
+ def fill_missing(
24
+ df: pd.DataFrame,
25
+ *,
26
+ strategy: Annotated[_MISSING_STRATEGY, "全局填充策略,默认中位数填充"] = "median",
27
+ column_strategy: Annotated[
28
+ dict[str, _MISSING_STRATEGY] | None, "为不同列配置不同的填充策略,不配置时,采用全局填充策略"
29
+ ] = None,
30
+ fill_value: Annotated[str, "strategy=fill 时的全局填充值"] = "Unknown",
31
+ column_fill_value: Annotated[
32
+ dict[str, str] | None, "strategy=fill 时,为不同列配置不同的填充值,不配置时,采用全局填充值"
33
+ ] = None,
34
+ group_by: Annotated[str | list[str] | None, "strategy=fill_grouped_mode 时,使用分组后的众数填充"] = None,
35
+ missing_indicator: Annotated[bool, r"增加缺失指示器, {col}_is_missing"] = True,
36
+ **kwargs,
37
+ ) -> pd.DataFrame:
38
+ """按列填充缺失值,missing_indicator=True 默认开启 {col}_is_missing 缺失指示列。
39
+
40
+ strategy 全局默认 median;column_strategy / column_fill_value 可对单列覆盖。fallback 路径:数值列遇到 fill/mode 时先回退到全局 strategy、二次失败才落到 median;分类列遇到 median/mean/interpolate/forward/backward 时直接 fallback 用 fill_value 填充。fill_grouped_mode 必须配 group_by,否则抛 EMPTY_GROUPY_BY。
41
+
42
+ 数值列:mean 用正态分布、近似无离群点的场景,median 用含离群点的稳健场景;forward/backward 用时序相邻填充场景,interpolate 用趋势平滑序列场景。分类列:fill 引入新类别保留缺失信号,mode 用全局众数、fill_grouped_mode 用分组众数。
43
+ """
44
+ column_strategy = column_strategy or {}
45
+ column_fill_value = column_fill_value or {}
46
+
47
+ cleaned_df = df
48
+
49
+ missing = cleaned_df.isnull()
50
+ missing_count = missing.sum()
51
+ missing_cols = missing_count[missing_count > 0].index
52
+
53
+ if missing_indicator:
54
+ indicator_df = missing[missing_cols].astype(int).add_suffix("_is_missing")
55
+ cleaned_df = pd.concat([df, indicator_df], axis=1)
56
+
57
+ def fill_numeric(df: pd.DataFrame, col, fallback_cnt: bool):
58
+ _strategy = column_strategy.get(str(col), strategy)
59
+ match _strategy:
60
+ case "mean":
61
+ df[col] = df[col].fillna(df[col].mean())
62
+ logger.debug(f"Filled {col} with mean")
63
+ case "median":
64
+ df[col] = df[col].fillna(df[col].median())
65
+ logger.debug(f"Filled {col} with median")
66
+ case "forward":
67
+ df[col] = df[col].ffill()
68
+ logger.debug(f"Filled {col} with ffill")
69
+ case "backward":
70
+ df[col] = df[col].bfill()
71
+ logger.debug(f"Filled {col} with bfill")
72
+ case "interpolate":
73
+ df[col] = df[col].interpolate(method="linear")
74
+ logger.debug(f"Filled {col} with linear interpolate")
75
+ case "fill_grouped_mode":
76
+ if not group_by:
77
+ raise EMPTY_GROUPY_BY
78
+ mode_map = df.groupby(group_by)[col].agg(lambda x: x.mode()[0] if not x.mode().empty else 1e-8)
79
+ df[col] = df[col].fillna(df[group_by].map(mode_map))
80
+ logger.debug(f"Filled {col} with fill_grouped_mode")
81
+ case _:
82
+ if fallback_cnt := not fallback_cnt:
83
+ logger.warning(
84
+ f"{col} 类型为 {df[col].dtype},不支持 {_strategy} 填充策略,回滚到 {strategy} 填充策略"
85
+ )
86
+ _strategy = strategy
87
+ fill_numeric(df, col, fallback_cnt)
88
+ else:
89
+ logger.warning(f"{col} 类型为 {df[col].dtype},不支持 {_strategy} 填充策略,回滚到 median 填充策略")
90
+ df[col] = df[col].fillna(df[col].median())
91
+ return df
92
+
93
+ def fill_category(df: pd.DataFrame, col, fallback_cnt: bool):
94
+ _strategy = column_strategy.get(str(col), strategy)
95
+ _fill_value = column_fill_value.get(str(col), fill_value)
96
+ match _strategy:
97
+ case "fill":
98
+ df[col] = df[col].fillna(_fill_value)
99
+ logger.debug(f"Filled {col} with {_fill_value}")
100
+ case "mode":
101
+ mode_0 = df[col].mode()[0]
102
+ df[col] = df[col].fillna(mode_0)
103
+ logger.debug(f"Filled {col} with {mode_0}")
104
+ case "fill_grouped_mode":
105
+ if not group_by:
106
+ raise EMPTY_GROUPY_BY
107
+ mode_map = df.groupby(group_by)[col].agg(lambda x: x.mode()[0] if not x.mode().empty else _fill_value)
108
+ df[col] = df[col].fillna(df[group_by].map(mode_map))
109
+ logger.debug(f"Filled {col} with fill_grouped_mode")
110
+ case _:
111
+ logger.warning(f"{col} 类型为 {df[col].dtype},不支持 {_strategy} 填充策略,回滚到 fill 填充策略")
112
+ df[col] = df[col].fillna(_fill_value)
113
+
114
+ for col in missing_cols:
115
+ fallback_cnt = False
116
+ if pd.api.types.is_numeric_dtype(cleaned_df[col]):
117
+ fill_numeric(cleaned_df, col, fallback_cnt)
118
+ else:
119
+ fill_category(cleaned_df, col, fallback_cnt)
120
+
121
+ return cleaned_df
@@ -0,0 +1,106 @@
1
+ """IQR 异常值检测器"""
2
+
3
+ import logging
4
+ from typing import Annotated, Literal
5
+
6
+ import numpy as np
7
+ import pandas as pd
8
+
9
+ from mflowy.driver.config import StepType
10
+ from mflowy.driver.handler import handler
11
+ from mflowy.middlewares.data_inject import inject_df
12
+ from mflowy.middlewares.log_df_diff import df_diff
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+
17
+ @handler(StepType.CLEAN, inject_df, df_diff)
18
+ def iqr_detector(
19
+ df: pd.DataFrame,
20
+ *,
21
+ threshold: Annotated[float, "IQR 倍数阈值"] = 1.5,
22
+ strategy: Annotated[Literal["remove", "cap", "replace"], "处理方式 (remove/cap/replace)"] = "remove",
23
+ replace_value: Annotated[float | None, "替换值(strategy=replace 时必填)"] = None,
24
+ columns: Annotated[list[str] | None, "检测列,None 表示所有数值列"] = None,
25
+ **kwargs,
26
+ ) -> pd.DataFrame:
27
+ """基于四分位距(IQR)检测数值列异常值并按 strategy 处理。
28
+
29
+ 按列计算 Q1/Q3,落在 [Q1 - threshold·IQR, Q3 + threshold·IQR] 之外的样本视为异常(threshold 默认 1.5)。columns=None 时检测所有数值列;非数值列与不存在的列被跳过。
30
+
31
+ strategy 三分支:remove(删除整行异常样本)、cap(把越界值夹到 IQR 边界,整数列自动转 float)、replace(把异常值替换为 replace_value,缺省抛 ValueError)。
32
+
33
+ iqr_detector 用"分布偏斜、有重尾"的非正态场景,zscore_detector 用"近似正态分布"场景。
34
+ """
35
+ result = df.copy()
36
+
37
+ # 确定检测列
38
+ if columns is None:
39
+ columns = result.select_dtypes(include=[np.number]).columns.tolist()
40
+
41
+ # ---- 检测阶段 ----
42
+ outlier_mask = np.zeros(len(result), dtype=bool)
43
+ skipped: list[str] = []
44
+
45
+ for col in columns:
46
+ if col not in result.columns:
47
+ skipped.append(col)
48
+ continue
49
+ if not pd.api.types.is_numeric_dtype(result[col]):
50
+ skipped.append(f"{col} (非数值类型)")
51
+ continue
52
+
53
+ Q1 = result[col].quantile(0.25)
54
+ Q3 = result[col].quantile(0.75)
55
+ IQR = Q3 - Q1
56
+ lower_bound = Q1 - threshold * IQR
57
+ upper_bound = Q3 + threshold * IQR
58
+
59
+ col_outliers = (result[col] < lower_bound) | (result[col] > upper_bound)
60
+ outlier_mask |= col_outliers
61
+
62
+ outlier_count = int(outlier_mask.sum())
63
+
64
+ if skipped:
65
+ logger.debug(f"跳过 {len(skipped)} 列: {skipped}")
66
+
67
+ logger.debug(f"Detected {outlier_count} outliers using IQR method (threshold={threshold})")
68
+
69
+ # ---- 处理阶段 ----
70
+ if strategy == "remove":
71
+ result = result[~outlier_mask].copy()
72
+ logger.info(f"Removed {outlier_count} outlier samples")
73
+
74
+ elif strategy == "cap":
75
+ # IQR 检测器的 cap 始终使用 IQR 边界
76
+ numeric_cols = result.select_dtypes(include=[np.number]).columns
77
+
78
+ for col in numeric_cols:
79
+ Q1 = result[col].quantile(0.25)
80
+ Q3 = result[col].quantile(0.75)
81
+ IQR = Q3 - Q1
82
+ lower_bound = Q1 - threshold * IQR
83
+ upper_bound = Q3 + threshold * IQR
84
+
85
+ if pd.api.types.is_integer_dtype(result[col]):
86
+ result[col] = result[col].astype(float)
87
+ result.loc[result[col] < lower_bound, col] = lower_bound
88
+ result.loc[result[col] > upper_bound, col] = upper_bound
89
+
90
+ logger.debug(f"Capped {outlier_count} outlier values to IQR boundary")
91
+
92
+ elif strategy == "replace":
93
+ if replace_value is None:
94
+ raise ValueError("replace_value 参数在 strategy='replace' 时必须提供")
95
+
96
+ numeric_cols = result.select_dtypes(include=[np.number]).columns
97
+
98
+ for col in numeric_cols:
99
+ result.loc[outlier_mask, col] = replace_value
100
+
101
+ logger.debug(f"Replaced {outlier_count} outlier values with {replace_value}")
102
+
103
+ else:
104
+ raise ValueError(f"未知的处理方式: {strategy}")
105
+
106
+ return result