InsurAutoML 0.2.8__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. insurautoml-0.3.0/InsurAutoML/VERSION +1 -0
  2. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/__init__.py +5 -83
  3. insurautoml-0.3.0/InsurAutoML/_optional.py +124 -0
  4. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/balancing/base.py +2 -3
  5. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/balancing/mixed_sampling.py +36 -25
  6. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/balancing/over_sampling.py +47 -33
  7. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/balancing/under_sampling.py +168 -95
  8. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/base.py +3 -4
  9. {insurautoml-0.2.8/InsurAutoML/ext/twinreduction/twinreduction → insurautoml-0.3.0/InsurAutoML/dashboard}/__init__.py +35 -6
  10. insurautoml-0.3.0/InsurAutoML/dashboard/__main__.py +121 -0
  11. insurautoml-0.3.0/InsurAutoML/dashboard/callback.py +154 -0
  12. insurautoml-0.3.0/InsurAutoML/dashboard/events.py +533 -0
  13. insurautoml-0.3.0/InsurAutoML/dashboard/layout.py +408 -0
  14. insurautoml-0.3.0/InsurAutoML/dashboard/server.py +337 -0
  15. insurautoml-0.3.0/InsurAutoML/dashboard/space.py +358 -0
  16. insurautoml-0.3.0/InsurAutoML/dashboard/static/app.js +872 -0
  17. insurautoml-0.3.0/InsurAutoML/dashboard/static/index.html +196 -0
  18. insurautoml-0.3.0/InsurAutoML/dashboard/static/style.css +482 -0
  19. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/datasets/__init__.py +1 -1
  20. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/datasets/image.py +5 -11
  21. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/datasets/tabular.py +39 -82
  22. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/datasets/text.py +3 -3
  23. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/encoding/encoding.py +56 -29
  24. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/ext/__init__.py +16 -2
  25. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/ext/suppsplit/__init__.py +1 -1
  26. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/ext/suppsplit/suppsplit/__init__.py +1 -2
  27. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/ext/suppsplit/suppsplit/split.py +65 -13
  28. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/ext/suppsplit/suppsplit/utils.py +4 -3
  29. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/ext/suppsplit/suppsplit_cpp/bindings.cpp +19 -10
  30. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/ext/suppsplit/suppsplit_cpp/sp.cpp +15 -7
  31. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/feature_selection/__init__.py +1 -44
  32. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/feature_selection/base.py +3 -3
  33. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/feature_selection/embed.py +22 -13
  34. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/feature_selection/filter.py +123 -45
  35. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/feature_selection/hybrid.py +68 -49
  36. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/feature_selection/sklearn.py +122 -119
  37. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/feature_selection/wrapper.py +53 -35
  38. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/hpo/DL.py +4 -4
  39. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/hpo/ML.py +73 -37
  40. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/hpo/__init__.py +0 -1
  41. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/hpo/base.py +985 -779
  42. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/hpo/ensemble.py +72 -37
  43. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/hpo/informed/ML.py +61 -35
  44. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/hpo/informed/__init__.py +24 -5
  45. insurautoml-0.3.0/InsurAutoML/hpo/informed/base.py +905 -0
  46. insurautoml-0.3.0/InsurAutoML/hpo/informed/fixed.py +276 -0
  47. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/hpo/informed/informed_ensemble.py +85 -38
  48. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/hpo/informed/utils.py +103 -63
  49. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/hpo/utils.py +141 -65
  50. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/hyperparameters/__init__.py +2 -2
  51. insurautoml-0.3.0/InsurAutoML/hyperparameters/ray/balancing_hyperparameter.py +115 -0
  52. insurautoml-0.3.0/InsurAutoML/hyperparameters/ray/classifier_hyperparameter.py +365 -0
  53. insurautoml-0.2.8/InsurAutoML/utils/_c/__init__.py → insurautoml-0.3.0/InsurAutoML/hyperparameters/ray/encoder_hyperparameter.py +31 -16
  54. insurautoml-0.3.0/InsurAutoML/hyperparameters/ray/feature_selection_hyperparameter.py +274 -0
  55. insurautoml-0.3.0/InsurAutoML/hyperparameters/ray/imputer_hyperparameter.py +109 -0
  56. insurautoml-0.3.0/InsurAutoML/hyperparameters/ray/prep_hyperparameter.py +68 -0
  57. insurautoml-0.3.0/InsurAutoML/hyperparameters/ray/regressor_hyperparameter.py +387 -0
  58. insurautoml-0.3.0/InsurAutoML/hyperparameters/ray/scaling_hyperparameter.py +80 -0
  59. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/imputation/__init__.py +1 -1
  60. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/imputation/base.py +29 -36
  61. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/imputation/clustering.py +109 -92
  62. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/imputation/multiple.py +44 -34
  63. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/imputation/nn.py +81 -65
  64. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/model/FNN.py +46 -32
  65. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/model/RNN.py +66 -36
  66. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/model/__init__.py +1 -88
  67. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/model/base.py +5 -8
  68. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/model/catboost.py +9 -1
  69. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/model/clustering.py +0 -1
  70. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/model/gam.py +4 -2
  71. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/model/lightgbm.py +21 -14
  72. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/model/sklearn_classifiers.py +176 -151
  73. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/model/sklearn_regressors.py +184 -113
  74. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/model/xgboost.py +14 -8
  75. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/prep/__init__.py +6 -1
  76. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/prep/pipeline.py +91 -76
  77. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/prep/prep.py +1 -1
  78. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/prep/utils.py +28 -22
  79. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/scaling/base.py +8 -4
  80. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/scaling/scaling.py +100 -84
  81. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/utils/__init__.py +0 -1
  82. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/utils/base.py +62 -32
  83. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/utils/data.py +125 -60
  84. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/utils/eda.py +22 -11
  85. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/utils/file.py +14 -5
  86. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/utils/metadata.py +49 -32
  87. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/utils/optimize.py +496 -265
  88. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/utils/stats.py +119 -61
  89. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/utils/tensor.py +7 -8
  90. insurautoml-0.3.0/InsurAutoML.egg-info/PKG-INFO +239 -0
  91. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML.egg-info/SOURCES.txt +21 -11
  92. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML.egg-info/requires.txt +16 -30
  93. insurautoml-0.3.0/PKG-INFO +239 -0
  94. insurautoml-0.3.0/README.md +171 -0
  95. insurautoml-0.3.0/pyproject.toml +174 -0
  96. insurautoml-0.3.0/setup.py +388 -0
  97. insurautoml-0.2.8/InsurAutoML/ext/twinreduction/__init__.py +0 -41
  98. insurautoml-0.2.8/InsurAutoML/ext/twinreduction/twinning_cpp/bindings.cpp +0 -101
  99. insurautoml-0.2.8/InsurAutoML/ext/twinreduction/twinning_cpp/twinning.cpp +0 -163
  100. insurautoml-0.2.8/InsurAutoML/ext/twinreduction/twinreduction/twinreduction.py +0 -131
  101. insurautoml-0.2.8/InsurAutoML/feature_selection/autosklearn.py +0 -644
  102. insurautoml-0.2.8/InsurAutoML/hpo/informed/base.py +0 -1445
  103. insurautoml-0.2.8/InsurAutoML/hpo/informed/fixed.py +0 -177
  104. insurautoml-0.2.8/InsurAutoML/model/autosklearn.py +0 -1552
  105. insurautoml-0.2.8/InsurAutoML/utils/_c/_base.pyx +0 -154
  106. insurautoml-0.2.8/InsurAutoML/utils/_c/_stat.pyx +0 -41
  107. insurautoml-0.2.8/InsurAutoML.egg-info/PKG-INFO +0 -342
  108. insurautoml-0.2.8/InsurAutoML.egg-info/not-zip-safe +0 -1
  109. insurautoml-0.2.8/PKG-INFO +0 -342
  110. insurautoml-0.2.8/README.md +0 -259
  111. insurautoml-0.2.8/setup.py +0 -603
  112. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/balancing/__init__.py +0 -0
  113. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/constant.py +0 -0
  114. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/encoding/__init__.py +0 -0
  115. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/encoding/base.py +0 -0
  116. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/ext/suppsplit/suppsplit_cpp/sPlit.cpp +0 -0
  117. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/model/ML.py +0 -0
  118. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/scaling/__init__.py +0 -0
  119. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML/version.py +0 -0
  120. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML.egg-info/dependency_links.txt +0 -0
  121. {insurautoml-0.2.8 → insurautoml-0.3.0}/InsurAutoML.egg-info/top_level.txt +0 -0
  122. {insurautoml-0.2.8 → insurautoml-0.3.0}/LICENSE +0 -0
  123. {insurautoml-0.2.8 → insurautoml-0.3.0}/setup.cfg +0 -0
  124. {insurautoml-0.2.8 → insurautoml-0.3.0}/tests/test_base.py +0 -0
@@ -0,0 +1 @@
1
+ 0.3.0
@@ -171,97 +171,19 @@ __version__ = _get_version()
171
171
  # "AutoNextWordPrediction": AutoNextWordPrediction,
172
172
  # }
173
173
 
174
+ # public names of the package (the submodules export the individual components,
175
+ # e.g. InsurAutoML.encoding, InsurAutoML.imputation, InsurAutoML.model)
174
176
  __all__ = [
175
177
  "load_data", # _base
176
178
  "no_processing",
177
- "random_guess", # _utils
178
- "random_index",
179
- "random_list",
180
- "is_date",
181
- "feature_rounding",
182
- "train_test_split",
183
- "minloc",
184
- "maxloc",
185
- "True_index",
186
- "nan_cov",
187
- "class_means",
188
- "empirical_covariance",
189
- "class_cov",
190
- "Pearson_Corr",
191
- "MI",
192
- "t_score",
193
- "ANOVA",
194
- "as_dataframe",
179
+ "set_seed",
180
+ "train_test_split", # _utils
195
181
  "type_of_task",
196
- "formatting",
197
- "Timer",
198
- "unify_nan",
199
- "remove_index_columns",
200
- "get_missing_matrix",
201
- "SimpleImputer", # _missing
202
- "DummyImputer",
203
- "JointImputer",
204
- "ExpectationMaximization",
205
- "KNNImputer",
206
- "MissForestImputer",
207
- "MICE",
208
- "GAIN",
209
- "AAI_kNN",
210
- "KMI",
211
- "CMI",
212
- "k_Prototype_NN",
213
- "DataEncoding", # _encoding
214
- "MinMaxScale", # _scaling
215
- "Standardize",
216
- "Normalize",
217
- "RobustScale",
218
- "PowerTransformer",
219
- "QuantileTransformer",
220
- "Winsorization",
221
- "Feature_Manipulation",
222
- "Feature_Truncation",
223
- "SimpleRandomOverSampling", # _imbalance
224
- "SimpleRandomUnderSampling",
225
- "TomekLink",
226
- "EditedNearestNeighbor",
227
- "CondensedNearestNeighbor",
228
- "OneSidedSelection",
229
- "CNN_TomekLink",
230
- "Smote",
231
- "Smote_TomekLink",
232
- "Smote_ENN",
233
- "LDASelection", # _feature_selection
234
- "PCA_FeatureSelection",
235
- "RBFSampler",
236
- "FeatureFilter",
237
- "ASFFS",
238
- "GeneticAlgorithm",
239
- "densifier", # from sklearn
240
- "extra_trees_preproc_for_classification",
241
- "extra_trees_preproc_for_regression",
242
- "fast_ica",
243
- "feature_agglomeration",
244
- "kernel_pca",
245
- "kitchen_sinks",
246
- "liblinear_svc_preprocessor",
247
- "nystroem_sampler",
248
- "pca",
249
- "polynomial",
250
- "random_trees_embedding",
251
- # 'select_percentile',
252
- "select_percentile_classification",
253
- "select_percentile_regression",
254
- "select_rates_classification",
255
- "select_rates_regression",
256
- "truncatedSVD",
257
- "classifiers", # _model
258
- "regressors",
182
+ "EDA",
259
183
  "AutoTabular", # _model_selection
260
184
  "AutoTabularClassifier",
261
185
  "AutoTabularRegressor",
262
186
  "InformedAutoTabular",
263
187
  "InformedAutoTabularRegressor",
264
188
  "InformedAutoTabularClassifier",
265
- "AutoTextClassifier",
266
- "AutoNextWordPrediction",
267
189
  ]
@@ -0,0 +1,124 @@
1
+ """
2
+ File Name: _optional.py
3
+ Author: Panyi Dong
4
+ GitHub: https://github.com/PanyiDong/
5
+ Actuarial and Risk Management Sciences, University of Illinois at Urbana-Champaign (UIUC)
6
+
7
+ Project: InsurAutoML
8
+ Latest Version: 0.2.9
9
+ Relative Path: /InsurAutoML/_optional.py
10
+ File Created: Monday, 16th November 2026 7:12:56 pm
11
+ Author: Panyi Dong (panyid2@illinois.edu)
12
+
13
+ -----
14
+ Last Modified: Monday, 16th November 2026 11:40:01 pm
15
+ Modified By: Panyi Dong (panyid2@illinois.edu)
16
+
17
+ -----
18
+ MIT License
19
+
20
+ Copyright (c) 2026, Panyi Dong
21
+
22
+ Permission is hereby granted, free of charge, to any person obtaining a copy
23
+ of this software and associated documentation files (the "Software"), to deal
24
+ in the Software without restriction, including without limitation the rights
25
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
26
+ copies of the Software, and to permit persons to whom the Software is
27
+ furnished to do so, subject to the following conditions:
28
+
29
+ The above copyright notice and this permission notice shall be included in all
30
+ copies or substantial portions of the Software.
31
+
32
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
33
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
34
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
35
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
36
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
37
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38
+ SOFTWARE.
39
+
40
+ Availability of the optional "informed" installation.
41
+
42
+ The informed pipelines (InformedAutoTabular*, InsurAutoML.prep) need the compiled
43
+ C++ extension in InsurAutoML.ext (SPlit) and the extra Python
44
+ dependencies (missforest, ...). They are only built/installed with
45
+
46
+ INFORMEDAUTOML=1 pip install "InsurAutoML[informed]"
47
+
48
+ The checks below do not import anything, so a default installation can import
49
+ InsurAutoML without the optional components.
50
+ """
51
+
52
+ from __future__ import annotations
53
+
54
+ import importlib.util
55
+ import os
56
+ from importlib.machinery import EXTENSION_SUFFIXES
57
+ from typing import List
58
+
59
+ INFORMED_INSTALL_CMD = 'INFORMEDAUTOML=1 pip install "InsurAutoML[informed]"'
60
+ # an existing (normal) installation is kept by pip as "already satisfied", reinstall
61
+ # InsurAutoML itself to build the C++ extension
62
+ INFORMED_REINSTALL_CMD = (
63
+ "INFORMEDAUTOML=1 pip install --force-reinstall --no-deps --no-cache-dir "
64
+ "InsurAutoML"
65
+ )
66
+
67
+ # Python packages only needed by the informed pipelines
68
+ INFORMED_PACKAGES = ["missforest"]
69
+
70
+ # compiled extensions: (sub-package of InsurAutoML.ext, extension module name)
71
+ INFORMED_EXTENSIONS = [
72
+ ("suppsplit", "suppsplit_cpp"),
73
+ ]
74
+
75
+ _EXT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ext")
76
+
77
+
78
+ class InformedNotInstalledError(ImportError):
79
+ """The optional informed installation is missing."""
80
+
81
+
82
+ def missing_informed_components() -> List[str]:
83
+ """Names of the missing informed components (empty if all are available)."""
84
+ missing = [
85
+ package
86
+ for package in INFORMED_PACKAGES
87
+ if importlib.util.find_spec(package) is None
88
+ ]
89
+ for subpackage, module in INFORMED_EXTENSIONS:
90
+ # look for the compiled extension file (e.g. suppsplit_cpp.cpython-312-*.so)
91
+ # without importing; the C++ source folder has the same name as the module
92
+ folder = os.path.join(_EXT_DIR, subpackage)
93
+ if not any(
94
+ os.path.isfile(os.path.join(folder, module + suffix))
95
+ for suffix in EXTENSION_SUFFIXES
96
+ ):
97
+ missing.append("InsurAutoML.ext.{}.{}".format(subpackage, module))
98
+
99
+ return missing
100
+
101
+
102
+ def informed_available() -> bool:
103
+ """Whether the informed pipelines can be used."""
104
+ return len(missing_informed_components()) == 0
105
+
106
+
107
+ def informed_not_installed_error(feature: str = "InformedAutoTabular") -> ImportError:
108
+ """Error asking the user to install the informed version."""
109
+ return InformedNotInstalledError(
110
+ "{} is not available, the InformedAutoML version is not installed "
111
+ "(missing: {}).\nInstall it with:\n {}\nIf InsurAutoML is already installed, "
112
+ "also rebuild it with the C++ extension:\n {}".format(
113
+ feature,
114
+ ", ".join(missing_informed_components()),
115
+ INFORMED_INSTALL_CMD,
116
+ INFORMED_REINSTALL_CMD,
117
+ )
118
+ )
119
+
120
+
121
+ def require_informed(feature: str = "InformedAutoTabular") -> None:
122
+ """Raise InformedNotInstalledError if the informed installation is missing."""
123
+ if not informed_available():
124
+ raise informed_not_installed_error(feature)
@@ -38,13 +38,12 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38
38
  SOFTWARE.
39
39
  """
40
40
 
41
- from typing import Union, Tuple
41
+ from typing import Optional, Union, Tuple
42
42
  import pandas as pd
43
43
  from ..constant import MAX_ITER
44
44
 
45
45
 
46
46
  class BaseBalancing:
47
-
48
47
  """
49
48
  Base class for balancing methods.
50
49
  """
@@ -56,6 +55,6 @@ class BaseBalancing:
56
55
  return MAX_ITER
57
56
 
58
57
  def fit_transform(
59
- self, X: pd.DataFrame, y: pd.DataFrame = None
58
+ self, X: pd.DataFrame, y: Optional[pd.DataFrame] = None
60
59
  ) -> Union[pd.DataFrame, Tuple[pd.DataFrame, pd.DataFrame]]:
61
60
  raise NotImplementedError
@@ -40,9 +40,10 @@ SOFTWARE.
40
40
 
41
41
  from __future__ import annotations
42
42
 
43
- from typing import Union, Tuple
43
+ from typing import Any, List, Optional, Union, Tuple, cast
44
44
  import numpy as np
45
45
  import pandas as pd
46
+ from ..utils.base import copy_frame
46
47
  import warnings
47
48
 
48
49
  from ..constant import MAX_ITER
@@ -61,7 +62,6 @@ balancing machine learning training data. ACM SIGKDD explorations newsletter, 6(
61
62
 
62
63
 
63
64
  class Smote_TomekLink(Smote, TomekLink):
64
-
65
65
  """
66
66
  Run Smote then run Tomek Link to balance dataset
67
67
  """
@@ -71,10 +71,10 @@ class Smote_TomekLink(Smote, TomekLink):
71
71
  imbalance_threshold: float = 0.9,
72
72
  norm: str = "l2",
73
73
  all: bool = True,
74
- max_iter: int = None,
74
+ max_iter: Optional[int] = None,
75
75
  k: int = 5,
76
76
  generation: str = "mean",
77
- seed: int = None,
77
+ seed: Optional[int] = None,
78
78
  ) -> None:
79
79
  self.imbalance_threshold = imbalance_threshold
80
80
  self.norm = norm
@@ -87,30 +87,33 @@ class Smote_TomekLink(Smote, TomekLink):
87
87
  self._fitted = False # whether the model has been fitted
88
88
 
89
89
  def fit_transform(
90
- self, X: pd.DataFrame, y: pd.DataFrame = None
90
+ self, X: pd.DataFrame, y: Optional[pd.DataFrame] = None
91
91
  ) -> Union[pd.DataFrame, Tuple[pd.DataFrame, pd.DataFrame]]:
92
92
  try: # if missing y, will be None value; or will be dataframe, use df.empty for judge
93
- _empty = y.empty
93
+ _empty = cast(pd.DataFrame, y).empty
94
94
  except AttributeError:
95
95
  _empty = y is None
96
96
 
97
97
  if (
98
98
  not _empty
99
99
  ): # if no y input, only convert X; or, combine X and y to consider balancing
100
+ assert y is not None # a None y has no .empty, so _empty is True
100
101
  features = list(X.columns)
101
- response = list(y.columns)
102
+ response: Optional[List[str]] = list(y.columns)
102
103
  data = pd.concat([X, y], axis=1)
103
104
  else:
104
105
  features = list(X.columns)
105
106
  response = None
106
107
  data = X
107
108
 
108
- _data = data.copy(deep=True)
109
+ _data = cast(pd.DataFrame, copy_frame(data, deep=True))
109
110
  if not is_imbalance(_data, self.imbalance_threshold):
110
111
  warnings.warn("The dataset is balanced, no change.")
111
112
  else:
113
+ # keep the user threshold, __init__ below overwrites it
114
+ _threshold = self.imbalance_threshold
112
115
  super().__init__(
113
- imbalance_threshold=(1.0 + self.imbalance_threshold) / 2,
116
+ imbalance_threshold=(1.0 + _threshold) / 2,
114
117
  norm=self.norm,
115
118
  all=self.all,
116
119
  max_iter=self.max_iter,
@@ -118,27 +121,29 @@ class Smote_TomekLink(Smote, TomekLink):
118
121
  k=self.k,
119
122
  generation=self.generation,
120
123
  )
121
- _data = super().fit_transform(_data)
124
+ _data = cast(pd.DataFrame, super().fit_transform(_data))
122
125
 
123
126
  super(Smote, self).__init__(
124
- imbalance_threshold=self.imbalance_threshold,
127
+ imbalance_threshold=_threshold,
125
128
  norm=self.norm,
126
129
  all=self.all,
127
130
  max_iter=self.max_iter,
128
131
  seed=self.seed,
129
132
  )
130
- _data = super(Smote, self).fit_transform(_data)
133
+ _data = cast(pd.DataFrame, super(Smote, self).fit_transform(_data))
134
+ self.imbalance_threshold = _threshold
131
135
 
132
136
  self._fitted = True
133
137
 
134
138
  if not _empty:
135
- return _data[features], _data[response]
139
+ return cast(pd.DataFrame, _data[features]), cast(
140
+ pd.DataFrame, _data[cast(List[str], response)]
141
+ )
136
142
  else:
137
143
  return _data
138
144
 
139
145
 
140
146
  class Smote_ENN(Smote, EditedNearestNeighbor):
141
-
142
147
  """
143
148
  Run Smote then run ENN to balance dataset
144
149
  """
@@ -148,8 +153,8 @@ class Smote_ENN(Smote, EditedNearestNeighbor):
148
153
  imbalance_threshold: float = 0.9,
149
154
  norm: str = "l2",
150
155
  all: bool = True,
151
- max_iter: int = None,
152
- seed: int = None,
156
+ max_iter: Optional[int] = None,
157
+ seed: Optional[int] = None,
153
158
  k: int = 5,
154
159
  generation: str = "mean",
155
160
  ) -> None:
@@ -164,31 +169,34 @@ class Smote_ENN(Smote, EditedNearestNeighbor):
164
169
  self._fitted = False # whether the model has been fitted
165
170
 
166
171
  def fit_transform(
167
- self, X: pd.DataFrame, y: pd.DataFrame = None
172
+ self, X: pd.DataFrame, y: Optional[pd.DataFrame] = None
168
173
  ) -> Union[pd.DataFrame, Tuple[pd.DataFrame, pd.DataFrame]]:
169
174
  try: # if missing y, will be None value; or will be dataframe, use df.empty for judge
170
- _empty = y.empty
175
+ _empty = cast(pd.DataFrame, y).empty
171
176
  except AttributeError:
172
177
  _empty = y is None
173
178
 
174
179
  if (
175
180
  not _empty
176
181
  ): # if no y input, only convert X; or, combine X and y to consider balancing
182
+ assert y is not None # a None y has no .empty, so _empty is True
177
183
  features = list(X.columns)
178
- response = list(y.columns)
184
+ response: Optional[List[str]] = list(y.columns)
179
185
  data = pd.concat([X, y], axis=1)
180
186
  else:
181
187
  features = list(X.columns)
182
188
  response = None
183
189
  data = X
184
190
 
185
- _data = data.copy(deep=True)
191
+ _data = cast(pd.DataFrame, copy_frame(data, deep=True))
186
192
 
187
193
  if not is_imbalance(_data, self.imbalance_threshold):
188
194
  warnings.warn("The dataset is balanced, no change.")
189
195
  else:
196
+ # keep the user threshold, __init__ below overwrites it
197
+ _threshold = self.imbalance_threshold
190
198
  super().__init__(
191
- imbalance_threshold=(1.0 + self.imbalance_threshold) / 2,
199
+ imbalance_threshold=(1.0 + _threshold) / 2,
192
200
  norm=self.norm,
193
201
  all=self.all,
194
202
  max_iter=self.max_iter,
@@ -196,20 +204,23 @@ class Smote_ENN(Smote, EditedNearestNeighbor):
196
204
  k=self.k,
197
205
  generation=self.generation,
198
206
  )
199
- _data = super().fit_transform(_data)
207
+ _data = cast(pd.DataFrame, super().fit_transform(_data))
200
208
 
201
209
  super(Smote, self).__init__(
202
- imbalance_threshold=self.imbalance_threshold,
210
+ imbalance_threshold=_threshold,
203
211
  all=self.all,
204
212
  max_iter=self.max_iter,
205
213
  seed=self.seed,
206
214
  k=self.k,
207
215
  )
208
- _data = super(Smote, self).fit_transform(_data)
216
+ _data = cast(pd.DataFrame, super(Smote, self).fit_transform(_data))
217
+ self.imbalance_threshold = _threshold
209
218
 
210
219
  self._fitted = True
211
220
 
212
221
  if not _empty:
213
- return _data[features], _data[response]
222
+ return cast(pd.DataFrame, _data[features]), cast(
223
+ pd.DataFrame, _data[cast(List[str], response)]
224
+ )
214
225
  else:
215
226
  return _data
@@ -40,9 +40,10 @@ SOFTWARE.
40
40
 
41
41
  from __future__ import annotations
42
42
 
43
- from typing import Union, Tuple
43
+ from typing import Any, List, Optional, Union, Tuple, cast
44
44
  import numpy as np
45
45
  import pandas as pd
46
+ from ..utils.base import copy_frame
46
47
  import warnings
47
48
  import sklearn
48
49
  import sklearn.utils
@@ -62,7 +63,6 @@ balancing machine learning training data. ACM SIGKDD explorations newsletter, 6(
62
63
 
63
64
 
64
65
  class SimpleRandomOverSampling(BaseBalancing):
65
-
66
66
  """
67
67
  Simple Random Over-Sampling
68
68
  Randomly draw samples from minority class and replicate the sample
@@ -83,8 +83,8 @@ class SimpleRandomOverSampling(BaseBalancing):
83
83
  self,
84
84
  imbalance_threshold: float = 0.9,
85
85
  all: bool = True,
86
- max_iter: int = None,
87
- seed: int = None,
86
+ max_iter: Optional[int] = None,
87
+ seed: Optional[int] = None,
88
88
  ) -> None:
89
89
  self.imbalance_threshold = imbalance_threshold
90
90
  self.all = all
@@ -95,25 +95,26 @@ class SimpleRandomOverSampling(BaseBalancing):
95
95
  self._fitted = False # whether the model has been fitted
96
96
 
97
97
  def fit_transform(
98
- self, X: pd.DataFrame, y: pd.DataFrame = None
98
+ self, X: pd.DataFrame, y: Optional[pd.DataFrame] = None
99
99
  ) -> Union[pd.DataFrame, Tuple[pd.DataFrame, pd.DataFrame]]:
100
100
  try: # if missing y, will be None value; or will be dataframe, use df.empty for judge
101
- _empty = y.empty
101
+ _empty = cast(pd.DataFrame, y).empty
102
102
  except AttributeError:
103
103
  _empty = y is None
104
104
 
105
105
  if (
106
106
  not _empty
107
107
  ): # if no y input, only convert X; or, combine X and y to consider balancing
108
+ assert y is not None # a None y has no .empty, so _empty is True
108
109
  features = list(X.columns)
109
- response = list(y.columns)
110
+ response: Optional[List[str]] = list(y.columns)
110
111
  data = pd.concat([X, y], axis=1)
111
112
  else:
112
113
  features = list(X.columns)
113
114
  response = None
114
115
  data = X
115
116
 
116
- _data = data.copy(deep=True)
117
+ _data = cast(pd.DataFrame, copy_frame(data, deep=True))
117
118
  if not is_imbalance(_data, self.imbalance_threshold):
118
119
  warnings.warn("The dataset is balanced, no change.")
119
120
  else:
@@ -126,18 +127,20 @@ class SimpleRandomOverSampling(BaseBalancing):
126
127
  self._fitted = True
127
128
 
128
129
  if not _empty: # return balanced X and y if y is also inputted
129
- return _data[features], _data[response]
130
+ return cast(pd.DataFrame, _data[features]), cast(
131
+ pd.DataFrame, _data[cast(List[str], response)]
132
+ )
130
133
  else:
131
134
  return _data
132
135
 
133
136
  def _fit_transform(
134
- self, X: Union[pd.DataFrame, Tuple[pd.DataFrame, pd.DataFrame]]
137
+ self, X: pd.DataFrame
135
138
  ) -> (
136
139
  pd.DataFrame
137
140
  ): # using random over-sampling to balance the first imbalanced feature
138
141
  n, features = len(X), list(X.columns)
139
- _imbalanced_feature, _majority = is_imbalance(
140
- X, self.imbalance_threshold, value=True
142
+ _imbalanced_feature, _majority = cast(
143
+ Tuple[str, Any], is_imbalance(X, self.imbalance_threshold, value=True)
141
144
  )
142
145
  _minority_class = X.loc[X[_imbalanced_feature] != _majority]
143
146
 
@@ -156,13 +159,14 @@ class SimpleRandomOverSampling(BaseBalancing):
156
159
  ]
157
160
  )
158
161
  # shuffle the data
159
- X = sklearn.utils.shuffle(X.reset_index(drop=True)).reset_index(drop=True)
162
+ X = cast(
163
+ pd.DataFrame, sklearn.utils.shuffle(X.reset_index(drop=True))
164
+ ).reset_index(drop=True)
160
165
 
161
166
  return X
162
167
 
163
168
 
164
169
  class Smote(BaseBalancing):
165
-
166
170
  """
167
171
  Synthetic Minority Over-sampling Technique (Smote)
168
172
  use over-sampling to generate minority class points using nearest neighbors
@@ -193,8 +197,8 @@ class Smote(BaseBalancing):
193
197
  imbalance_threshold: float = 0.9,
194
198
  norm: str = "l2",
195
199
  all: bool = True,
196
- max_iter: int = None,
197
- seed: int = None,
200
+ max_iter: Optional[int] = None,
201
+ seed: Optional[int] = None,
198
202
  k: int = 5,
199
203
  generation: str = "mean",
200
204
  ) -> None:
@@ -210,45 +214,59 @@ class Smote(BaseBalancing):
210
214
  self._fitted = False # whether the model has been fitted
211
215
 
212
216
  def fit_transform(
213
- self, X: pd.DataFrame, y: pd.DataFrame = None
217
+ self, X: pd.DataFrame, y: Optional[pd.DataFrame] = None
214
218
  ) -> Union[pd.DataFrame, Tuple[pd.DataFrame, pd.DataFrame]]:
215
219
  try: # if missing y, will be None value; or will be dataframe, use df.empty for judge
216
- _empty = y.empty
220
+ _empty = cast(pd.DataFrame, y).empty
217
221
  except AttributeError:
218
222
  _empty = y is None
219
223
 
220
224
  if (
221
225
  not _empty
222
226
  ): # if no y input, only convert X; or, combine X and y to consider balancing
227
+ assert y is not None # a None y has no .empty, so _empty is True
223
228
  features = list(X.columns)
224
- response = list(y.columns)
229
+ response: Optional[List[str]] = list(y.columns)
225
230
  data = pd.concat([X, y], axis=1)
226
231
  else:
227
232
  features = list(X.columns)
228
233
  response = None
229
234
  data = X
230
235
 
231
- _data = data.copy(deep=True)
236
+ _data = cast(pd.DataFrame, copy_frame(data, deep=True))
232
237
  if not is_imbalance(_data, self.imbalance_threshold):
233
238
  warnings.warn("The dataset is balanced, no change.")
234
239
  else:
235
240
  if self.all:
236
- while is_imbalance(_data, self.imbalance_threshold):
237
- _data = self._fit_transform(_data)
241
+ # stop when a pass does not change the data (can not balance further)
242
+ _iter = 0
243
+ while (
244
+ is_imbalance(_data, self.imbalance_threshold)
245
+ and _iter < self.max_iter
246
+ ):
247
+ _new_data = Smote._fit_transform(self, _data)
248
+ if len(_new_data) == len(_data):
249
+ warnings.warn("Smote can not reduce the imbalance further.")
250
+ _data = _new_data
251
+ break
252
+ _data = _new_data
253
+ _iter += 1
238
254
  else:
239
- _data = self._fit_transform(_data)
255
+ _data = Smote._fit_transform(self, _data)
240
256
 
241
257
  self._fitted = True
242
258
 
243
259
  if not _empty:
244
- return _data[features], _data[response]
260
+ return cast(pd.DataFrame, _data[features]), cast(
261
+ pd.DataFrame, _data[cast(List[str], response)]
262
+ )
245
263
  else:
246
264
  return _data
247
265
 
248
266
  def _fit_transform(self, X: pd.DataFrame) -> pd.DataFrame:
249
267
  n, features = len(X), list(X.columns)
250
- _imbalanced_feature, _majority = is_imbalance(
251
- X, self.imbalance_threshold, value=True
268
+ _imbalanced_feature, _majority = cast(
269
+ Tuple[str, Any], is_imbalance(X, self.imbalance_threshold, value=True)
252
270
  )
253
271
  _minority_class = X.loc[X[_imbalanced_feature] != _majority]
254
272
 
@@ -271,14 +289,10 @@ class Smote(BaseBalancing):
271
289
  ].mean(axis=0)
272
290
  elif self.generation == "random":
273
291
  X.loc[len(X), :] = X.loc[_sample.index, :] + np.random.rand() * (
274
- X.loc[X.index[_link], :] - X.lox[_sample.index, :]
292
+ X.loc[X.index[_link], :] - X.loc[_sample.index, :]
275
293
  )
276
294
  else:
277
- raise ValueError(
278
- 'Not recognizing generation method! Should be in \
279
- ["mean", "random"], get {}'.format(
280
- self.generation
281
- )
282
- )
295
+ raise ValueError('Not recognizing generation method! Should be in \
296
+ ["mean", "random"], get {}'.format(self.generation))
283
297
 
284
298
  return X