InsurAutoML 0.2.8__tar.gz → 0.2.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- insurautoml-0.2.9/InsurAutoML/VERSION +1 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/__init__.py +5 -83
- insurautoml-0.2.9/InsurAutoML/_optional.py +94 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/balancing/mixed_sampling.py +10 -4
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/balancing/over_sampling.py +14 -3
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/balancing/under_sampling.py +79 -29
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/base.py +1 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/datasets/image.py +2 -9
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/datasets/tabular.py +0 -43
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/datasets/text.py +0 -1
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/encoding/encoding.py +22 -14
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/ext/__init__.py +16 -2
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/ext/suppsplit/__init__.py +1 -1
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/ext/suppsplit/suppsplit/__init__.py +1 -1
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/ext/suppsplit/suppsplit/split.py +8 -2
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/ext/suppsplit/suppsplit/utils.py +4 -3
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/ext/suppsplit/suppsplit_cpp/bindings.cpp +10 -7
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/feature_selection/__init__.py +10 -53
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/feature_selection/base.py +3 -3
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/feature_selection/filter.py +15 -2
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/feature_selection/hybrid.py +9 -2
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/feature_selection/sklearn.py +11 -5
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/feature_selection/wrapper.py +5 -4
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hpo/ML.py +17 -17
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hpo/base.py +345 -377
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hpo/informed/ML.py +24 -25
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hpo/informed/__init__.py +24 -5
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hpo/informed/base.py +166 -390
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hpo/informed/fixed.py +37 -33
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hpo/informed/informed_ensemble.py +47 -22
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hpo/informed/utils.py +74 -58
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hpo/utils.py +123 -60
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hyperparameters/__init__.py +2 -2
- insurautoml-0.2.9/InsurAutoML/hyperparameters/ray/balancing_hyperparameter.py +115 -0
- insurautoml-0.2.9/InsurAutoML/hyperparameters/ray/classifier_hyperparameter.py +365 -0
- insurautoml-0.2.8/InsurAutoML/utils/_c/__init__.py → insurautoml-0.2.9/InsurAutoML/hyperparameters/ray/encoder_hyperparameter.py +31 -16
- insurautoml-0.2.9/InsurAutoML/hyperparameters/ray/feature_selection_hyperparameter.py +274 -0
- insurautoml-0.2.9/InsurAutoML/hyperparameters/ray/imputer_hyperparameter.py +109 -0
- insurautoml-0.2.8/InsurAutoML/ext/twinreduction/twinreduction/__init__.py → insurautoml-0.2.9/InsurAutoML/hyperparameters/ray/prep_hyperparameter.py +32 -6
- insurautoml-0.2.9/InsurAutoML/hyperparameters/ray/regressor_hyperparameter.py +387 -0
- insurautoml-0.2.9/InsurAutoML/hyperparameters/ray/scaling_hyperparameter.py +80 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/imputation/base.py +1 -1
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/imputation/clustering.py +9 -6
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/imputation/multiple.py +14 -15
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/imputation/nn.py +34 -23
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/model/FNN.py +1 -1
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/model/RNN.py +9 -6
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/model/__init__.py +0 -87
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/model/catboost.py +9 -1
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/model/gam.py +4 -2
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/model/sklearn_classifiers.py +5 -6
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/model/sklearn_regressors.py +12 -3
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/prep/__init__.py +6 -1
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/prep/pipeline.py +61 -70
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/prep/prep.py +1 -1
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/prep/utils.py +25 -19
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/scaling/scaling.py +6 -8
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/utils/base.py +48 -28
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/utils/data.py +61 -21
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/utils/eda.py +12 -7
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/utils/file.py +11 -3
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/utils/metadata.py +24 -20
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/utils/optimize.py +319 -234
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/utils/stats.py +1 -2
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML.egg-info/PKG-INFO +64 -54
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML.egg-info/SOURCES.txt +52 -12
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML.egg-info/requires.txt +13 -30
- insurautoml-0.2.9/InsurAutoML.egg-info/top_level.txt +5 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/PKG-INFO +64 -54
- {insurautoml-0.2.8 → insurautoml-0.2.9}/README.md +291 -259
- insurautoml-0.2.9/pyproject.toml +120 -0
- insurautoml-0.2.9/setup.py +379 -0
- insurautoml-0.2.9/tests/test_balancing/test_balancing.py +72 -0
- insurautoml-0.2.9/tests/test_datasets/test_datasets_hf.py +164 -0
- insurautoml-0.2.9/tests/test_datasets/test_datasets_tabular.py +343 -0
- insurautoml-0.2.9/tests/test_encoder/test_encoder.py +95 -0
- insurautoml-0.2.9/tests/test_feature_selection/test_feature_selection.py +261 -0
- insurautoml-0.2.9/tests/test_feature_selection/test_feature_selection_more.py +736 -0
- insurautoml-0.2.9/tests/test_feature_selection/test_feature_selection_wrapper.py +226 -0
- insurautoml-0.2.9/tests/test_hpo/test_hpo.py +434 -0
- insurautoml-0.2.9/tests/test_hpo/test_hpo_2.py +435 -0
- insurautoml-0.2.9/tests/test_hpo/test_hpo_3.py +531 -0
- insurautoml-0.2.9/tests/test_hpo/test_hpo_ensemble.py +212 -0
- insurautoml-0.2.9/tests/test_hpo/test_hpo_ensemble_voting.py +237 -0
- insurautoml-0.2.9/tests/test_hpo/test_hpo_spaces.py +147 -0
- insurautoml-0.2.9/tests/test_hyperparameters/test_hyperparameters.py +109 -0
- insurautoml-0.2.9/tests/test_imputer/test_imputer.py +245 -0
- insurautoml-0.2.9/tests/test_imputer/test_imputer_clustering.py +381 -0
- insurautoml-0.2.9/tests/test_imputer/test_imputer_more.py +424 -0
- insurautoml-0.2.9/tests/test_imputer/test_imputer_nn.py +257 -0
- insurautoml-0.2.9/tests/test_informed/conftest.py +110 -0
- insurautoml-0.2.9/tests/test_informed/test_informed_automl.py +172 -0
- insurautoml-0.2.9/tests/test_informed/test_informed_ensemble.py +185 -0
- insurautoml-0.2.9/tests/test_informed/test_informed_objective.py +217 -0
- insurautoml-0.2.9/tests/test_informed/test_informed_pipeline.py +138 -0
- insurautoml-0.2.9/tests/test_informed/test_informed_prep.py +175 -0
- insurautoml-0.2.9/tests/test_model/test_model.py +208 -0
- insurautoml-0.2.9/tests/test_model/test_model_2.py +350 -0
- insurautoml-0.2.9/tests/test_model/test_model_nn.py +343 -0
- insurautoml-0.2.9/tests/test_scaling/test_scaling.py +148 -0
- insurautoml-0.2.8/InsurAutoML/utils/_c/_stat.pyx → insurautoml-0.2.9/tests/test_utils/test_metadata.py +36 -4
- insurautoml-0.2.9/tests/test_utils/test_optional.py +151 -0
- insurautoml-0.2.9/tests/test_utils/test_utils.py +126 -0
- insurautoml-0.2.9/tests/test_utils/test_utils_base.py +223 -0
- insurautoml-0.2.9/tests/test_utils/test_utils_data.py +218 -0
- insurautoml-0.2.9/tests/test_utils/test_utils_data_extra.py +398 -0
- insurautoml-0.2.9/tests/test_utils/test_utils_metadata_extra.py +290 -0
- insurautoml-0.2.9/tests/test_utils/test_utils_optimize.py +222 -0
- insurautoml-0.2.9/tests/test_utils/test_utils_optimize_more.py +272 -0
- insurautoml-0.2.9/tests/test_utils/test_utils_stat.py +189 -0
- insurautoml-0.2.9/tests/test_utils/test_utils_stats_extra.py +288 -0
- insurautoml-0.2.9/tests/test_utils/test_utils_tensor.py +143 -0
- insurautoml-0.2.8/InsurAutoML/ext/twinreduction/__init__.py +0 -41
- insurautoml-0.2.8/InsurAutoML/ext/twinreduction/twinning_cpp/bindings.cpp +0 -101
- insurautoml-0.2.8/InsurAutoML/ext/twinreduction/twinning_cpp/twinning.cpp +0 -163
- insurautoml-0.2.8/InsurAutoML/ext/twinreduction/twinreduction/twinreduction.py +0 -131
- insurautoml-0.2.8/InsurAutoML/feature_selection/autosklearn.py +0 -644
- insurautoml-0.2.8/InsurAutoML/model/autosklearn.py +0 -1552
- insurautoml-0.2.8/InsurAutoML/utils/_c/_base.pyx +0 -154
- insurautoml-0.2.8/InsurAutoML.egg-info/not-zip-safe +0 -1
- insurautoml-0.2.8/InsurAutoML.egg-info/top_level.txt +0 -1
- insurautoml-0.2.8/setup.py +0 -603
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/balancing/__init__.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/balancing/base.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/constant.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/datasets/__init__.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/encoding/__init__.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/encoding/base.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/ext/suppsplit/suppsplit_cpp/sPlit.cpp +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/ext/suppsplit/suppsplit_cpp/sp.cpp +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/feature_selection/embed.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hpo/DL.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hpo/__init__.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/hpo/ensemble.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/imputation/__init__.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/model/ML.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/model/base.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/model/clustering.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/model/lightgbm.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/model/xgboost.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/scaling/__init__.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/scaling/base.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/utils/__init__.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/utils/tensor.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML/version.py +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/InsurAutoML.egg-info/dependency_links.txt +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/LICENSE +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/setup.cfg +0 -0
- {insurautoml-0.2.8 → insurautoml-0.2.9}/tests/test_base.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.2.9
|
|
@@ -171,97 +171,19 @@ __version__ = _get_version()
|
|
|
171
171
|
# "AutoNextWordPrediction": AutoNextWordPrediction,
|
|
172
172
|
# }
|
|
173
173
|
|
|
174
|
+
# public names of the package (the submodules export the individual components,
|
|
175
|
+
# e.g. InsurAutoML.encoding, InsurAutoML.imputation, InsurAutoML.model)
|
|
174
176
|
__all__ = [
|
|
175
177
|
"load_data", # _base
|
|
176
178
|
"no_processing",
|
|
177
|
-
"
|
|
178
|
-
"
|
|
179
|
-
"random_list",
|
|
180
|
-
"is_date",
|
|
181
|
-
"feature_rounding",
|
|
182
|
-
"train_test_split",
|
|
183
|
-
"minloc",
|
|
184
|
-
"maxloc",
|
|
185
|
-
"True_index",
|
|
186
|
-
"nan_cov",
|
|
187
|
-
"class_means",
|
|
188
|
-
"empirical_covariance",
|
|
189
|
-
"class_cov",
|
|
190
|
-
"Pearson_Corr",
|
|
191
|
-
"MI",
|
|
192
|
-
"t_score",
|
|
193
|
-
"ANOVA",
|
|
194
|
-
"as_dataframe",
|
|
179
|
+
"set_seed",
|
|
180
|
+
"train_test_split", # _utils
|
|
195
181
|
"type_of_task",
|
|
196
|
-
"
|
|
197
|
-
"Timer",
|
|
198
|
-
"unify_nan",
|
|
199
|
-
"remove_index_columns",
|
|
200
|
-
"get_missing_matrix",
|
|
201
|
-
"SimpleImputer", # _missing
|
|
202
|
-
"DummyImputer",
|
|
203
|
-
"JointImputer",
|
|
204
|
-
"ExpectationMaximization",
|
|
205
|
-
"KNNImputer",
|
|
206
|
-
"MissForestImputer",
|
|
207
|
-
"MICE",
|
|
208
|
-
"GAIN",
|
|
209
|
-
"AAI_kNN",
|
|
210
|
-
"KMI",
|
|
211
|
-
"CMI",
|
|
212
|
-
"k_Prototype_NN",
|
|
213
|
-
"DataEncoding", # _encoding
|
|
214
|
-
"MinMaxScale", # _scaling
|
|
215
|
-
"Standardize",
|
|
216
|
-
"Normalize",
|
|
217
|
-
"RobustScale",
|
|
218
|
-
"PowerTransformer",
|
|
219
|
-
"QuantileTransformer",
|
|
220
|
-
"Winsorization",
|
|
221
|
-
"Feature_Manipulation",
|
|
222
|
-
"Feature_Truncation",
|
|
223
|
-
"SimpleRandomOverSampling", # _imbalance
|
|
224
|
-
"SimpleRandomUnderSampling",
|
|
225
|
-
"TomekLink",
|
|
226
|
-
"EditedNearestNeighbor",
|
|
227
|
-
"CondensedNearestNeighbor",
|
|
228
|
-
"OneSidedSelection",
|
|
229
|
-
"CNN_TomekLink",
|
|
230
|
-
"Smote",
|
|
231
|
-
"Smote_TomekLink",
|
|
232
|
-
"Smote_ENN",
|
|
233
|
-
"LDASelection", # _feature_selection
|
|
234
|
-
"PCA_FeatureSelection",
|
|
235
|
-
"RBFSampler",
|
|
236
|
-
"FeatureFilter",
|
|
237
|
-
"ASFFS",
|
|
238
|
-
"GeneticAlgorithm",
|
|
239
|
-
"densifier", # from sklearn
|
|
240
|
-
"extra_trees_preproc_for_classification",
|
|
241
|
-
"extra_trees_preproc_for_regression",
|
|
242
|
-
"fast_ica",
|
|
243
|
-
"feature_agglomeration",
|
|
244
|
-
"kernel_pca",
|
|
245
|
-
"kitchen_sinks",
|
|
246
|
-
"liblinear_svc_preprocessor",
|
|
247
|
-
"nystroem_sampler",
|
|
248
|
-
"pca",
|
|
249
|
-
"polynomial",
|
|
250
|
-
"random_trees_embedding",
|
|
251
|
-
# 'select_percentile',
|
|
252
|
-
"select_percentile_classification",
|
|
253
|
-
"select_percentile_regression",
|
|
254
|
-
"select_rates_classification",
|
|
255
|
-
"select_rates_regression",
|
|
256
|
-
"truncatedSVD",
|
|
257
|
-
"classifiers", # _model
|
|
258
|
-
"regressors",
|
|
182
|
+
"EDA",
|
|
259
183
|
"AutoTabular", # _model_selection
|
|
260
184
|
"AutoTabularClassifier",
|
|
261
185
|
"AutoTabularRegressor",
|
|
262
186
|
"InformedAutoTabular",
|
|
263
187
|
"InformedAutoTabularRegressor",
|
|
264
188
|
"InformedAutoTabularClassifier",
|
|
265
|
-
"AutoTextClassifier",
|
|
266
|
-
"AutoNextWordPrediction",
|
|
267
189
|
]
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""
|
|
2
|
+
File: _optional.py
|
|
3
|
+
Author: Panyi Dong
|
|
4
|
+
GitHub: https://github.com/PanyiDong/
|
|
5
|
+
Mathematics Department, University of Illinois at Urbana-Champaign (UIUC)
|
|
6
|
+
|
|
7
|
+
Project: InsurAutoML
|
|
8
|
+
Relative Path: /InsurAutoML/_optional.py
|
|
9
|
+
|
|
10
|
+
Availability of the optional "informed" installation.
|
|
11
|
+
|
|
12
|
+
The informed pipelines (InformedAutoTabular*, InsurAutoML.prep) need the compiled
|
|
13
|
+
C++ extension in InsurAutoML.ext (SPlit) and the extra Python
|
|
14
|
+
dependencies (missforest, ...). They are only built/installed with
|
|
15
|
+
|
|
16
|
+
INFORMEDAUTOML=1 pip install "InsurAutoML[informed]"
|
|
17
|
+
|
|
18
|
+
The checks below do not import anything, so a default installation can import
|
|
19
|
+
InsurAutoML without the optional components.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import importlib.util
|
|
25
|
+
import os
|
|
26
|
+
from importlib.machinery import EXTENSION_SUFFIXES
|
|
27
|
+
from typing import List
|
|
28
|
+
|
|
29
|
+
INFORMED_INSTALL_CMD = 'INFORMEDAUTOML=1 pip install "InsurAutoML[informed]"'
|
|
30
|
+
# an existing (normal) installation is kept by pip as "already satisfied", reinstall
|
|
31
|
+
# InsurAutoML itself to build the C++ extension
|
|
32
|
+
INFORMED_REINSTALL_CMD = (
|
|
33
|
+
"INFORMEDAUTOML=1 pip install --force-reinstall --no-deps --no-cache-dir "
|
|
34
|
+
"InsurAutoML"
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
# Python packages only needed by the informed pipelines
|
|
38
|
+
INFORMED_PACKAGES = ["missforest"]
|
|
39
|
+
|
|
40
|
+
# compiled extensions: (sub-package of InsurAutoML.ext, extension module name)
|
|
41
|
+
INFORMED_EXTENSIONS = [
|
|
42
|
+
("suppsplit", "suppsplit_cpp"),
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
_EXT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ext")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class InformedNotInstalledError(ImportError):
|
|
49
|
+
"""The optional informed installation is missing."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def missing_informed_components() -> List[str]:
|
|
53
|
+
"""Names of the missing informed components (empty if all are available)."""
|
|
54
|
+
missing = [
|
|
55
|
+
package
|
|
56
|
+
for package in INFORMED_PACKAGES
|
|
57
|
+
if importlib.util.find_spec(package) is None
|
|
58
|
+
]
|
|
59
|
+
for subpackage, module in INFORMED_EXTENSIONS:
|
|
60
|
+
# look for the compiled extension file (e.g. suppsplit_cpp.cpython-312-*.so)
|
|
61
|
+
# without importing; the C++ source folder has the same name as the module
|
|
62
|
+
folder = os.path.join(_EXT_DIR, subpackage)
|
|
63
|
+
if not any(
|
|
64
|
+
os.path.isfile(os.path.join(folder, module + suffix))
|
|
65
|
+
for suffix in EXTENSION_SUFFIXES
|
|
66
|
+
):
|
|
67
|
+
missing.append("InsurAutoML.ext.{}.{}".format(subpackage, module))
|
|
68
|
+
|
|
69
|
+
return missing
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def informed_available() -> bool:
|
|
73
|
+
"""Whether the informed pipelines can be used."""
|
|
74
|
+
return len(missing_informed_components()) == 0
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def informed_not_installed_error(feature: str = "InformedAutoTabular") -> ImportError:
|
|
78
|
+
"""Error asking the user to install the informed version."""
|
|
79
|
+
return InformedNotInstalledError(
|
|
80
|
+
"{} is not available, the InformedAutoML version is not installed "
|
|
81
|
+
"(missing: {}).\nInstall it with:\n {}\nIf InsurAutoML is already installed, "
|
|
82
|
+
"also rebuild it with the C++ extension:\n {}".format(
|
|
83
|
+
feature,
|
|
84
|
+
", ".join(missing_informed_components()),
|
|
85
|
+
INFORMED_INSTALL_CMD,
|
|
86
|
+
INFORMED_REINSTALL_CMD,
|
|
87
|
+
)
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def require_informed(feature: str = "InformedAutoTabular") -> None:
|
|
92
|
+
"""Raise InformedNotInstalledError if the informed installation is missing."""
|
|
93
|
+
if not informed_available():
|
|
94
|
+
raise informed_not_installed_error(feature)
|
|
@@ -109,8 +109,10 @@ class Smote_TomekLink(Smote, TomekLink):
|
|
|
109
109
|
if not is_imbalance(_data, self.imbalance_threshold):
|
|
110
110
|
warnings.warn("The dataset is balanced, no change.")
|
|
111
111
|
else:
|
|
112
|
+
# keep the user threshold, __init__ below overwrites it
|
|
113
|
+
_threshold = self.imbalance_threshold
|
|
112
114
|
super().__init__(
|
|
113
|
-
imbalance_threshold=(1.0 +
|
|
115
|
+
imbalance_threshold=(1.0 + _threshold) / 2,
|
|
114
116
|
norm=self.norm,
|
|
115
117
|
all=self.all,
|
|
116
118
|
max_iter=self.max_iter,
|
|
@@ -121,13 +123,14 @@ class Smote_TomekLink(Smote, TomekLink):
|
|
|
121
123
|
_data = super().fit_transform(_data)
|
|
122
124
|
|
|
123
125
|
super(Smote, self).__init__(
|
|
124
|
-
imbalance_threshold=
|
|
126
|
+
imbalance_threshold=_threshold,
|
|
125
127
|
norm=self.norm,
|
|
126
128
|
all=self.all,
|
|
127
129
|
max_iter=self.max_iter,
|
|
128
130
|
seed=self.seed,
|
|
129
131
|
)
|
|
130
132
|
_data = super(Smote, self).fit_transform(_data)
|
|
133
|
+
self.imbalance_threshold = _threshold
|
|
131
134
|
|
|
132
135
|
self._fitted = True
|
|
133
136
|
|
|
@@ -187,8 +190,10 @@ class Smote_ENN(Smote, EditedNearestNeighbor):
|
|
|
187
190
|
if not is_imbalance(_data, self.imbalance_threshold):
|
|
188
191
|
warnings.warn("The dataset is balanced, no change.")
|
|
189
192
|
else:
|
|
193
|
+
# keep the user threshold, __init__ below overwrites it
|
|
194
|
+
_threshold = self.imbalance_threshold
|
|
190
195
|
super().__init__(
|
|
191
|
-
imbalance_threshold=(1.0 +
|
|
196
|
+
imbalance_threshold=(1.0 + _threshold) / 2,
|
|
192
197
|
norm=self.norm,
|
|
193
198
|
all=self.all,
|
|
194
199
|
max_iter=self.max_iter,
|
|
@@ -199,13 +204,14 @@ class Smote_ENN(Smote, EditedNearestNeighbor):
|
|
|
199
204
|
_data = super().fit_transform(_data)
|
|
200
205
|
|
|
201
206
|
super(Smote, self).__init__(
|
|
202
|
-
imbalance_threshold=
|
|
207
|
+
imbalance_threshold=_threshold,
|
|
203
208
|
all=self.all,
|
|
204
209
|
max_iter=self.max_iter,
|
|
205
210
|
seed=self.seed,
|
|
206
211
|
k=self.k,
|
|
207
212
|
)
|
|
208
213
|
_data = super(Smote, self).fit_transform(_data)
|
|
214
|
+
self.imbalance_threshold = _threshold
|
|
209
215
|
|
|
210
216
|
self._fitted = True
|
|
211
217
|
|
|
@@ -233,10 +233,21 @@ class Smote(BaseBalancing):
|
|
|
233
233
|
warnings.warn("The dataset is balanced, no change.")
|
|
234
234
|
else:
|
|
235
235
|
if self.all:
|
|
236
|
-
|
|
237
|
-
|
|
236
|
+
# stop when a pass does not change the data (can not balance further)
|
|
237
|
+
_iter = 0
|
|
238
|
+
while (
|
|
239
|
+
is_imbalance(_data, self.imbalance_threshold)
|
|
240
|
+
and _iter < self.max_iter
|
|
241
|
+
):
|
|
242
|
+
_new_data = Smote._fit_transform(self, _data)
|
|
243
|
+
if len(_new_data) == len(_data):
|
|
244
|
+
warnings.warn("Smote can not reduce the imbalance further.")
|
|
245
|
+
_data = _new_data
|
|
246
|
+
break
|
|
247
|
+
_data = _new_data
|
|
248
|
+
_iter += 1
|
|
238
249
|
else:
|
|
239
|
-
_data =
|
|
250
|
+
_data = Smote._fit_transform(self, _data)
|
|
240
251
|
|
|
241
252
|
self._fitted = True
|
|
242
253
|
|
|
@@ -221,10 +221,21 @@ class TomekLink(BaseBalancing):
|
|
|
221
221
|
warnings.warn("The dataset is balanced, no change.")
|
|
222
222
|
else:
|
|
223
223
|
if self.all:
|
|
224
|
-
|
|
225
|
-
|
|
224
|
+
# stop when a pass does not change the data (can not balance further)
|
|
225
|
+
_iter = 0
|
|
226
|
+
while (
|
|
227
|
+
is_imbalance(_data, self.imbalance_threshold)
|
|
228
|
+
and _iter < self.max_iter
|
|
229
|
+
):
|
|
230
|
+
_new_data = TomekLink._fit_transform(self, _data)
|
|
231
|
+
if len(_new_data) == len(_data):
|
|
232
|
+
warnings.warn("TomekLink can not reduce the imbalance further.")
|
|
233
|
+
_data = _new_data
|
|
234
|
+
break
|
|
235
|
+
_data = _new_data
|
|
236
|
+
_iter += 1
|
|
226
237
|
else:
|
|
227
|
-
_data =
|
|
238
|
+
_data = TomekLink._fit_transform(self, _data)
|
|
228
239
|
|
|
229
240
|
self._fitted = True
|
|
230
241
|
|
|
@@ -341,10 +352,23 @@ class EditedNearestNeighbor(BaseBalancing):
|
|
|
341
352
|
warnings.warn("The dataset is balanced, no change.")
|
|
342
353
|
else:
|
|
343
354
|
if self.all:
|
|
344
|
-
|
|
345
|
-
|
|
355
|
+
# stop when a pass does not change the data (can not balance further)
|
|
356
|
+
_iter = 0
|
|
357
|
+
while (
|
|
358
|
+
is_imbalance(_data, self.imbalance_threshold)
|
|
359
|
+
and _iter < self.max_iter
|
|
360
|
+
):
|
|
361
|
+
_new_data = EditedNearestNeighbor._fit_transform(self, _data)
|
|
362
|
+
if len(_new_data) == len(_data):
|
|
363
|
+
warnings.warn(
|
|
364
|
+
"EditedNearestNeighbor can not reduce the imbalance further."
|
|
365
|
+
)
|
|
366
|
+
_data = _new_data
|
|
367
|
+
break
|
|
368
|
+
_data = _new_data
|
|
369
|
+
_iter += 1
|
|
346
370
|
else:
|
|
347
|
-
_data =
|
|
371
|
+
_data = EditedNearestNeighbor._fit_transform(self, _data)
|
|
348
372
|
|
|
349
373
|
self._fitted = True
|
|
350
374
|
|
|
@@ -462,10 +486,24 @@ class CondensedNearestNeighbor(BaseBalancing):
|
|
|
462
486
|
warnings.warn("The dataset is balanced, no change.")
|
|
463
487
|
else:
|
|
464
488
|
if self.all:
|
|
465
|
-
|
|
466
|
-
|
|
489
|
+
# CNN only removes samples, stop when a pass removes none
|
|
490
|
+
# (the remaining imbalance can not be reduced by CNN)
|
|
491
|
+
_iter = 0
|
|
492
|
+
while (
|
|
493
|
+
is_imbalance(_data, self.imbalance_threshold)
|
|
494
|
+
and _iter < self.max_iter
|
|
495
|
+
):
|
|
496
|
+
_new_data = CondensedNearestNeighbor._fit_transform(self, _data)
|
|
497
|
+
if len(_new_data) == len(_data):
|
|
498
|
+
warnings.warn(
|
|
499
|
+
"CondensedNearestNeighbor can not reduce the imbalance further."
|
|
500
|
+
)
|
|
501
|
+
_data = _new_data
|
|
502
|
+
break
|
|
503
|
+
_data = _new_data
|
|
504
|
+
_iter += 1
|
|
467
505
|
else:
|
|
468
|
-
_data =
|
|
506
|
+
_data = CondensedNearestNeighbor._fit_transform(self, _data)
|
|
469
507
|
|
|
470
508
|
self._fitted = True
|
|
471
509
|
|
|
@@ -497,6 +535,9 @@ class CondensedNearestNeighbor(BaseBalancing):
|
|
|
497
535
|
X, self.imbalance_threshold, value=True
|
|
498
536
|
)
|
|
499
537
|
ini_neigh = self._get_initial_neigh(X[_imbalanced_feature])
|
|
538
|
+
# unique index, rows are tracked by index labels below
|
|
539
|
+
X = X.reset_index(drop=True)
|
|
540
|
+
_features = X.columns != _imbalanced_feature
|
|
500
541
|
|
|
501
542
|
_seed = self.seed
|
|
502
543
|
_iter = 0
|
|
@@ -507,26 +548,29 @@ class CondensedNearestNeighbor(BaseBalancing):
|
|
|
507
548
|
):
|
|
508
549
|
_minority_class = X.loc[X[_imbalanced_feature] != _majority]
|
|
509
550
|
_majority_class = X.loc[X[_imbalanced_feature] == _majority]
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
551
|
+
# subset: all minority samples + one random majority sample
|
|
552
|
+
_sample = _majority_class.sample(n=1, random_state=_seed)
|
|
553
|
+
_subset = pd.concat([_minority_class, _sample])
|
|
554
|
+
# majority samples not in the subset, to be checked by 1-NN
|
|
555
|
+
_rest = _majority_class.drop(index=_sample.index)
|
|
556
|
+
if _rest.empty:
|
|
557
|
+
X = _subset.reset_index(drop=True)
|
|
558
|
+
break
|
|
559
|
+
|
|
513
560
|
neigh = ini_neigh(n_neighbors=1)
|
|
514
561
|
neigh.fit(
|
|
515
|
-
_subset.loc[:,
|
|
562
|
+
_subset.loc[:, _features],
|
|
516
563
|
_subset[_imbalanced_feature],
|
|
517
564
|
)
|
|
518
|
-
y_predict = neigh.predict(
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
]
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
X = pd.concat([_subset, X.iloc[_not_matching_index, :]]).reset_index(
|
|
528
|
-
drop=True
|
|
529
|
-
)
|
|
565
|
+
y_predict = neigh.predict(_rest.loc[:, _features])
|
|
566
|
+
# keep the misclassified samples (by labels, not positions)
|
|
567
|
+
_misclassified = _rest.loc[
|
|
568
|
+
np.asarray(y_predict) != _rest[_imbalanced_feature].values
|
|
569
|
+
]
|
|
570
|
+
# no sample removed, condensed set reached
|
|
571
|
+
if len(_subset) + len(_misclassified) == len(X):
|
|
572
|
+
break
|
|
573
|
+
X = pd.concat([_subset, _misclassified]).reset_index(drop=True)
|
|
530
574
|
_seed += 1
|
|
531
575
|
_iter += 1
|
|
532
576
|
|
|
@@ -594,8 +638,10 @@ class OneSidedSelection(TomekLink, CondensedNearestNeighbor):
|
|
|
594
638
|
if not is_imbalance(_data, self.imbalance_threshold):
|
|
595
639
|
warnings.warn("The dataset is balanced, no change.")
|
|
596
640
|
else:
|
|
641
|
+
# keep the user threshold, __init__ below overwrites it
|
|
642
|
+
_threshold = self.imbalance_threshold
|
|
597
643
|
super().__init__(
|
|
598
|
-
imbalance_threshold=(1.0 +
|
|
644
|
+
imbalance_threshold=(1.0 + _threshold) / 2,
|
|
599
645
|
norm=self.norm,
|
|
600
646
|
all=self.all,
|
|
601
647
|
max_iter=self.max_iter,
|
|
@@ -604,12 +650,13 @@ class OneSidedSelection(TomekLink, CondensedNearestNeighbor):
|
|
|
604
650
|
_data = super().fit_transform(_data)
|
|
605
651
|
|
|
606
652
|
super(TomekLink, self).__init__(
|
|
607
|
-
imbalance_threshold=
|
|
653
|
+
imbalance_threshold=_threshold,
|
|
608
654
|
all=self.all,
|
|
609
655
|
max_iter=self.max_iter,
|
|
610
656
|
seed=self.seed,
|
|
611
657
|
)
|
|
612
658
|
_data = super(TomekLink, self).fit_transform(_data)
|
|
659
|
+
self.imbalance_threshold = _threshold
|
|
613
660
|
|
|
614
661
|
self._fitted = True
|
|
615
662
|
|
|
@@ -680,8 +727,10 @@ class CNN_TomekLink(CondensedNearestNeighbor, TomekLink):
|
|
|
680
727
|
if not is_imbalance(_data, self.imbalance_threshold):
|
|
681
728
|
warnings.warn("The dataset is balanced, no change.")
|
|
682
729
|
else:
|
|
730
|
+
# keep the user threshold, __init__ below overwrites it
|
|
731
|
+
_threshold = self.imbalance_threshold
|
|
683
732
|
super().__init__(
|
|
684
|
-
imbalance_threshold=(1.0 +
|
|
733
|
+
imbalance_threshold=(1.0 + _threshold) / 2,
|
|
685
734
|
all=self.all,
|
|
686
735
|
max_iter=self.max_iter,
|
|
687
736
|
seed=self.seed,
|
|
@@ -689,13 +738,14 @@ class CNN_TomekLink(CondensedNearestNeighbor, TomekLink):
|
|
|
689
738
|
_data = super().fit_transform(_data)
|
|
690
739
|
|
|
691
740
|
super(CondensedNearestNeighbor, self).__init__(
|
|
692
|
-
imbalance_threshold=
|
|
741
|
+
imbalance_threshold=_threshold,
|
|
693
742
|
norm=self.norm,
|
|
694
743
|
all=self.all,
|
|
695
744
|
max_iter=self.max_iter,
|
|
696
745
|
seed=self.seed,
|
|
697
746
|
)
|
|
698
747
|
_data = super(CondensedNearestNeighbor, self).fit_transform(_data)
|
|
748
|
+
self.imbalance_threshold = _threshold
|
|
699
749
|
|
|
700
750
|
self._fitted = True
|
|
701
751
|
|
|
@@ -49,6 +49,7 @@ import warnings
|
|
|
49
49
|
# check if rpy2 available
|
|
50
50
|
# if not, can not read R type data
|
|
51
51
|
import importlib
|
|
52
|
+
|
|
52
53
|
# NOTE: Remove rda/rdata loading as rpy2 sometimes causes installation issues
|
|
53
54
|
# Uncomment the following lines if R data loading is needed
|
|
54
55
|
# rpy2_spec = importlib.util.find_spec("rpy2")
|
|
@@ -46,7 +46,6 @@ from datasets import load_dataset
|
|
|
46
46
|
|
|
47
47
|
|
|
48
48
|
def CIFAR(train=True, test=False, version=10):
|
|
49
|
-
|
|
50
49
|
if version == 10:
|
|
51
50
|
dataname = "cifar10"
|
|
52
51
|
elif version == 100:
|
|
@@ -79,15 +78,9 @@ def MNIST(train=True, test=False):
|
|
|
79
78
|
|
|
80
79
|
def IMAGENET(train=True, test=False):
|
|
81
80
|
if train and not test:
|
|
82
|
-
dataset = load_dataset(
|
|
83
|
-
"imagenet-1k",
|
|
84
|
-
use_auth_token=True,
|
|
85
|
-
split="train")
|
|
81
|
+
dataset = load_dataset("imagenet-1k", use_auth_token=True, split="train")
|
|
86
82
|
elif test and not train:
|
|
87
|
-
dataset = load_dataset(
|
|
88
|
-
"imagenet-1k",
|
|
89
|
-
use_auth_token=True,
|
|
90
|
-
split="test")
|
|
83
|
+
dataset = load_dataset("imagenet-1k", use_auth_token=True, split="test")
|
|
91
84
|
elif train and test:
|
|
92
85
|
dataset = load_dataset("imagenet-1k", use_auth_token=True)
|
|
93
86
|
else:
|