medpipe 0.1.2.dev0__py3-none-any.whl → 0.2.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- medpipe/__init__.py +2 -1
- medpipe/_types.py +50 -0
- medpipe/data/__init__.py +4 -3
- medpipe/data/db.py +12 -10
- medpipe/data/preprocessing.py +66 -184
- medpipe/data/{Preprocessor.py → preprocessor.py} +65 -53
- medpipe/data/sampler.py +363 -214
- medpipe/data/utils.py +269 -0
- medpipe/data/weighting.py +77 -67
- medpipe/metrics/core.py +106 -138
- medpipe/metrics/plots.py +54 -33
- medpipe/models/__init__.py +1 -1
- medpipe/models/calibrators.py +345 -0
- medpipe/models/core.py +76 -54
- medpipe/models/predictors.py +280 -0
- medpipe/pipeline/__init__.py +2 -2
- medpipe/pipeline/pipeline.py +822 -0
- medpipe/py.typed +0 -0
- medpipe/utils/config.py +18 -11
- medpipe/utils/exceptions.py +23 -12
- medpipe/utils/io.py +11 -5
- medpipe/utils/logger.py +13 -6
- {medpipe-0.1.2.dev0.dist-info → medpipe-0.2.0.dev0.dist-info}/METADATA +7 -3
- medpipe-0.2.0.dev0.dist-info/RECORD +29 -0
- medpipe/models/Calibrator.py +0 -147
- medpipe/models/Predictor.py +0 -144
- medpipe/pipeline/Pipeline.py +0 -832
- medpipe-0.1.2.dev0.dist-info/RECORD +0 -26
- {medpipe-0.1.2.dev0.dist-info → medpipe-0.2.0.dev0.dist-info}/WHEEL +0 -0
- {medpipe-0.1.2.dev0.dist-info → medpipe-0.2.0.dev0.dist-info}/licenses/LICENSE +0 -0
- {medpipe-0.1.2.dev0.dist-info → medpipe-0.2.0.dev0.dist-info}/top_level.txt +0 -0
medpipe/__init__.py
CHANGED
|
@@ -21,7 +21,7 @@ from .metrics import (
|
|
|
21
21
|
print_metrics_CI,
|
|
22
22
|
)
|
|
23
23
|
from .models import get_full_proba, get_positive_proba, load_pipeline, save_pipeline
|
|
24
|
-
from .pipeline.
|
|
24
|
+
from .pipeline.pipeline import Pipeline
|
|
25
25
|
from .utils import (
|
|
26
26
|
exception_handler,
|
|
27
27
|
load_data_from_csv,
|
|
@@ -52,3 +52,4 @@ __all__ = [
|
|
|
52
52
|
"extract_data_from_db",
|
|
53
53
|
"extract_labels",
|
|
54
54
|
]
|
|
55
|
+
__version__ = "v0.2.0.dev0"
|
medpipe/_types.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Medpipe typings.
|
|
3
|
+
|
|
4
|
+
This module provides special types defined for the medpipe package.
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import TYPE_CHECKING, Annotated, Literal, Sequence, TypeAlias, TypeVar
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
import numpy.typing as npt
|
|
14
|
+
import pandas as pd
|
|
15
|
+
from sklearn.ensemble import HistGradientBoostingClassifier
|
|
16
|
+
from sklearn.isotonic import IsotonicRegression
|
|
17
|
+
from sklearn.linear_model import LogisticRegression
|
|
18
|
+
from sklearn.preprocessing import OrdinalEncoder, PowerTransformer, StandardScaler
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from medpipe.models.calibrators import IsotonicCalibrator, LogisticCalibrator
|
|
22
|
+
from medpipe.models.predictors import HGBClassifier
|
|
23
|
+
|
|
24
|
+
# Define data and label types
|
|
25
|
+
SingleClassLabels: TypeAlias = Annotated[npt.NDArray[np.integer], Literal["N"]]
|
|
26
|
+
MultiClassLabels: TypeAlias = Annotated[npt.NDArray[np.integer], Literal["N", "C"]]
|
|
27
|
+
FullProba: TypeAlias = Annotated[npt.NDArray[np.number], Literal["C", "N", 2]]
|
|
28
|
+
PosProba: TypeAlias = Annotated[npt.NDArray[np.number], Literal["N", "C"]]
|
|
29
|
+
Labels: TypeAlias = SingleClassLabels | MultiClassLabels
|
|
30
|
+
PredData: TypeAlias = pd.DataFrame | npt.NDArray # Predictor class data
|
|
31
|
+
Data: TypeAlias = PredData | PosProba # Pipeline class data
|
|
32
|
+
|
|
33
|
+
# Define preprocessing types
|
|
34
|
+
PreprocessOpConfig: TypeAlias = dict[str, dict[str, list[str]]]
|
|
35
|
+
PreprocessOp: TypeAlias = OrdinalEncoder | StandardScaler | PowerTransformer
|
|
36
|
+
|
|
37
|
+
# Define metric types
|
|
38
|
+
CI: TypeAlias = tuple[npt.NDArray, npt.NDArray, npt.NDArray]
|
|
39
|
+
CIDict: TypeAlias = dict[str, CI]
|
|
40
|
+
MetricDict: TypeAlias = dict[str, list[float]]
|
|
41
|
+
ModelMetrics: TypeAlias = dict[str, dict[str, Sequence[float]]]
|
|
42
|
+
|
|
43
|
+
# Define model types
|
|
44
|
+
Classifier: TypeAlias = HistGradientBoostingClassifier
|
|
45
|
+
Predictor: TypeAlias = "HGBClassifier"
|
|
46
|
+
Regressor: TypeAlias = LogisticRegression | IsotonicRegression
|
|
47
|
+
Calibrator: TypeAlias = "IsotonicCalibrator | LogisticCalibrator"
|
|
48
|
+
Model: TypeAlias = Classifier | Regressor
|
|
49
|
+
R = TypeVar("R", bound=Regressor) # Generic Type Variable for Regressors
|
|
50
|
+
C = TypeVar("C", bound=Classifier) # Generic Type Variable for Classifiers
|
medpipe/data/__init__.py
CHANGED
|
@@ -6,9 +6,10 @@ submodules:
|
|
|
6
6
|
- preprocessing: contains functions for preprocessing data.
|
|
7
7
|
- weighting: contains functions to compute sample weights for imbalanced data.
|
|
8
8
|
- sampler: contains functions to sample the data for imbalanced data.
|
|
9
|
-
-
|
|
9
|
+
- preprocessor: class to prepare data for fitting.
|
|
10
|
+
- utils: utility functions for data manipulation.
|
|
10
11
|
"""
|
|
11
12
|
|
|
12
|
-
from . import
|
|
13
|
+
from . import db, preprocessing, preprocessor, sampler, utils, weighting
|
|
13
14
|
from .db import extract_data_from_db
|
|
14
|
-
from .
|
|
15
|
+
from .utils import extract_labels
|
medpipe/data/db.py
CHANGED
|
@@ -8,11 +8,13 @@ Functions:
|
|
|
8
8
|
- extract_data_from_db: Queries a SQL .db to extract data.
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
11
13
|
import sqlite3
|
|
12
14
|
|
|
13
15
|
import pandas as pd
|
|
14
16
|
|
|
15
|
-
|
|
17
|
+
from medpipe.utils.exceptions import file_checks
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") -> None:
|
|
@@ -21,11 +23,11 @@ def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") ->
|
|
|
21
23
|
|
|
22
24
|
Parameters
|
|
23
25
|
----------
|
|
24
|
-
parquet_file
|
|
26
|
+
parquet_file : str
|
|
25
27
|
File path to the .parquet file.
|
|
26
|
-
db_file
|
|
28
|
+
db_file : str
|
|
27
29
|
File path to the .db file.
|
|
28
|
-
table_name : default: 'main'
|
|
30
|
+
table_name : str, default: 'main'
|
|
29
31
|
Name of the table to create in the SQL database.
|
|
30
32
|
|
|
31
33
|
Returns
|
|
@@ -48,12 +50,12 @@ def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") ->
|
|
|
48
50
|
|
|
49
51
|
"""
|
|
50
52
|
try:
|
|
51
|
-
|
|
53
|
+
file_checks(parquet_file, ".parquet")
|
|
52
54
|
except (FileNotFoundError, IsADirectoryError, TypeError, ValueError):
|
|
53
55
|
raise
|
|
54
56
|
|
|
55
57
|
try:
|
|
56
|
-
|
|
58
|
+
file_checks(db_file, ".db", exists=False)
|
|
57
59
|
except (FileNotFoundError, TypeError, ValueError, IsADirectoryError):
|
|
58
60
|
raise
|
|
59
61
|
|
|
@@ -69,7 +71,7 @@ def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") ->
|
|
|
69
71
|
conn.close()
|
|
70
72
|
|
|
71
73
|
|
|
72
|
-
def extract_data_from_db(db_file: str, query: str):
|
|
74
|
+
def extract_data_from_db(db_file: str, query: str) -> pd.DataFrame:
|
|
73
75
|
"""
|
|
74
76
|
Extracts data from a .db and saves it to a .csv file.
|
|
75
77
|
|
|
@@ -77,9 +79,9 @@ def extract_data_from_db(db_file: str, query: str):
|
|
|
77
79
|
|
|
78
80
|
Parameters
|
|
79
81
|
----------
|
|
80
|
-
db_file
|
|
82
|
+
db_file : str
|
|
81
83
|
Path to the .db file.
|
|
82
|
-
query
|
|
84
|
+
query : str
|
|
83
85
|
Query to send to extract data.
|
|
84
86
|
|
|
85
87
|
Returns
|
|
@@ -100,7 +102,7 @@ def extract_data_from_db(db_file: str, query: str):
|
|
|
100
102
|
|
|
101
103
|
"""
|
|
102
104
|
try:
|
|
103
|
-
|
|
105
|
+
file_checks(db_file, ".db")
|
|
104
106
|
except (FileNotFoundError, IsADirectoryError, TypeError, ValueError):
|
|
105
107
|
raise
|
|
106
108
|
|
medpipe/data/preprocessing.py
CHANGED
|
@@ -5,24 +5,29 @@ This module provides functions to preprocess data before training.
|
|
|
5
5
|
|
|
6
6
|
Functions:
|
|
7
7
|
- train_test_it: Creates a KFold iterator to split data into test and train sets.
|
|
8
|
-
- get_validation_idx: Removes some of the indices to create a validation set.
|
|
9
8
|
- convert_object_to_categorical: Converts object columns to categoricals.
|
|
10
9
|
- fit_preprocess_operations: Fits processing operations to data.
|
|
11
10
|
- bin_score: Bins the M3 score into 5 categories.
|
|
12
|
-
- extract_labels: Extracts prediction labels from data.
|
|
13
11
|
"""
|
|
14
12
|
|
|
15
|
-
from
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from typing import TYPE_CHECKING, Any, Mapping
|
|
16
16
|
|
|
17
17
|
import numpy as np
|
|
18
18
|
import pandas as pd
|
|
19
|
-
|
|
19
|
+
from sklearn.model_selection import GroupKFold, StratifiedKFold
|
|
20
20
|
from sklearn.preprocessing import OrdinalEncoder, PowerTransformer, StandardScaler
|
|
21
21
|
|
|
22
|
-
from medpipe.
|
|
22
|
+
from medpipe._types import PreprocessOp, PreprocessOpConfig
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
import numpy.typing as npt
|
|
23
26
|
|
|
24
27
|
|
|
25
|
-
def train_test_it(
|
|
28
|
+
def train_test_it(
|
|
29
|
+
group_k_fold: bool = False, **kwargs: Any
|
|
30
|
+
) -> StratifiedKFold | GroupKFold:
|
|
26
31
|
"""
|
|
27
32
|
Creates a KFold iterator to split data into test and train sets.
|
|
28
33
|
|
|
@@ -31,12 +36,12 @@ def train_test_it(group_k_fold=False, **kwargs):
|
|
|
31
36
|
group_k_fold : bool, default: False
|
|
32
37
|
If True, the data will be split using a group and a
|
|
33
38
|
GroupKFold iterator is returned.
|
|
34
|
-
**kwargs
|
|
39
|
+
**kwargs: Any
|
|
35
40
|
Extra arguments for the StratifiedKFold or GroupKFold class.
|
|
36
41
|
|
|
37
42
|
Returns
|
|
38
43
|
-------
|
|
39
|
-
kfold_it : StratifiedKFold
|
|
44
|
+
kfold_it : StratifiedKFold | GroupKFold
|
|
40
45
|
KFold iterator.
|
|
41
46
|
|
|
42
47
|
Raises
|
|
@@ -66,103 +71,20 @@ def train_test_it(group_k_fold=False, **kwargs):
|
|
|
66
71
|
args_dict.update({key: value})
|
|
67
72
|
|
|
68
73
|
if not group_k_fold:
|
|
69
|
-
kfold_it =
|
|
74
|
+
kfold_it = StratifiedKFold(**args_dict)
|
|
70
75
|
else:
|
|
71
|
-
kfold_it =
|
|
76
|
+
kfold_it = GroupKFold(**args_dict)
|
|
72
77
|
|
|
73
78
|
return kfold_it
|
|
74
79
|
|
|
75
80
|
|
|
76
|
-
def get_validation_idx(idx_list, groups=None, group_vals=None, val_size=0.1):
|
|
77
|
-
"""
|
|
78
|
-
Removes some of the indices to create a validation set.
|
|
79
|
-
|
|
80
|
-
If groups are provided and group_vals is None, all the indices of
|
|
81
|
-
the group with the largest value are selected as the validation set.
|
|
82
|
-
If group_vals is specified with groups, then the group_vals are
|
|
83
|
-
selected as the validation/test set.
|
|
84
|
-
|
|
85
|
-
Parameters
|
|
86
|
-
----------
|
|
87
|
-
idx_list : np.array(n_samples,)
|
|
88
|
-
Indices of the set to split.
|
|
89
|
-
groups : pd.Series(n_samples,) or np.ndarray or None, default: None
|
|
90
|
-
Groups to which the train indices belong.
|
|
91
|
-
group_vals : list[] or None, default: None
|
|
92
|
-
Group values that should be in the test set.
|
|
93
|
-
val_size : float, default: 0.1
|
|
94
|
-
Size of the validation set if groups are None.
|
|
95
|
-
|
|
96
|
-
Returns
|
|
97
|
-
-------
|
|
98
|
-
train_idx : np.array
|
|
99
|
-
Train indices.
|
|
100
|
-
val_idx : np.array
|
|
101
|
-
Validation indices.
|
|
102
|
-
|
|
103
|
-
Raises
|
|
104
|
-
------
|
|
105
|
-
TypeError
|
|
106
|
-
If groups is not pd.Series or np.ndarray.
|
|
107
|
-
If group_vals is not iterable.
|
|
108
|
-
If group_vals is not a list or a np.ndarray.
|
|
109
|
-
If val_size is not a float.
|
|
110
|
-
ValueError
|
|
111
|
-
If val_size < 0 or val_size > 1.
|
|
112
|
-
|
|
113
|
-
"""
|
|
114
|
-
array_check(idx_list)
|
|
115
|
-
if groups is not None:
|
|
116
|
-
# If groups are provided
|
|
117
|
-
if isinstance(groups, pd.Series):
|
|
118
|
-
groups = groups.to_numpy() # Convert to array
|
|
119
|
-
elif not isinstance(groups, np.ndarray):
|
|
120
|
-
raise TypeError(
|
|
121
|
-
f"groups should be a pd.Series or np.array, but got {type(groups)}"
|
|
122
|
-
)
|
|
123
|
-
array_check(idx_list)
|
|
124
|
-
array_dim_check(idx_list, groups, dim=0)
|
|
125
|
-
|
|
126
|
-
if group_vals is not None:
|
|
127
|
-
if not hasattr(group_vals, "__iter__"):
|
|
128
|
-
raise TypeError("group_vals should be iterable")
|
|
129
|
-
if (
|
|
130
|
-
isinstance(group_vals, dict)
|
|
131
|
-
or isinstance(group_vals, str)
|
|
132
|
-
or isinstance(group_vals, tuple)
|
|
133
|
-
):
|
|
134
|
-
raise TypeError("group_vals should be list or array")
|
|
135
|
-
val_idx = np.array([], dtype=np.int64)
|
|
136
|
-
train_idx = []
|
|
137
|
-
for group in group_vals:
|
|
138
|
-
val_idx = np.concatenate((val_idx, np.where(groups == group)[0]))
|
|
139
|
-
|
|
140
|
-
train_idx = np.setdiff1d(np.arange(len(groups)), val_idx)
|
|
141
|
-
|
|
142
|
-
else:
|
|
143
|
-
group_max = np.max(groups)
|
|
144
|
-
val_idx = np.where(groups == group_max)[0]
|
|
145
|
-
train_idx = np.where(groups != group_max)[0]
|
|
146
|
-
|
|
147
|
-
else:
|
|
148
|
-
if not isinstance(val_size, float):
|
|
149
|
-
raise TypeError(f"val_size should be a float, but got {type(val_size)}")
|
|
150
|
-
if val_size < 0.0 or val_size > 1.0:
|
|
151
|
-
raise ValueError(f"val_size should be between 0 and 1, but got {val_size}")
|
|
152
|
-
train_idx, val_idx = skl.model_selection.train_test_split(
|
|
153
|
-
idx_list, test_size=val_size, random_state=42
|
|
154
|
-
)
|
|
155
|
-
|
|
156
|
-
return train_idx, val_idx
|
|
157
|
-
|
|
158
|
-
|
|
159
81
|
def convert_object_to_categorical(data: pd.DataFrame) -> pd.DataFrame:
|
|
160
82
|
"""
|
|
161
83
|
Converts all object columns of a DataFrame to categoricals.
|
|
162
84
|
|
|
163
85
|
Parameters
|
|
164
86
|
----------
|
|
165
|
-
data
|
|
87
|
+
data : pd.DataFrame
|
|
166
88
|
DataFrame to manipulate.
|
|
167
89
|
|
|
168
90
|
Returns
|
|
@@ -176,19 +98,22 @@ def convert_object_to_categorical(data: pd.DataFrame) -> pd.DataFrame:
|
|
|
176
98
|
If data is not a pd.DataFrame.
|
|
177
99
|
|
|
178
100
|
"""
|
|
179
|
-
if
|
|
180
|
-
raise TypeError(f"data should be a pd.DataFrame, but got {type(data)}")
|
|
101
|
+
if not isinstance(data, pd.DataFrame):
|
|
102
|
+
raise TypeError(f"data should be a pd.DataFrame, but got {type(data).__name__}")
|
|
181
103
|
|
|
182
|
-
|
|
183
|
-
|
|
104
|
+
processed_data = data.copy()
|
|
105
|
+
obj_cols = processed_data.select_dtypes(include=["object"]).columns
|
|
184
106
|
|
|
185
|
-
for
|
|
186
|
-
|
|
107
|
+
# Vectorized conversion for all object columns at once
|
|
108
|
+
if not obj_cols.empty:
|
|
109
|
+
processed_data[obj_cols] = processed_data[obj_cols].astype("category")
|
|
187
110
|
|
|
188
111
|
return processed_data
|
|
189
112
|
|
|
190
113
|
|
|
191
|
-
def fit_preprocess_operations(
|
|
114
|
+
def fit_preprocess_operations(
|
|
115
|
+
data: pd.DataFrame, preprocessing_dict: PreprocessOpConfig
|
|
116
|
+
) -> Mapping[str, PreprocessOp | str]:
|
|
192
117
|
"""
|
|
193
118
|
Fits processing operations to data.
|
|
194
119
|
|
|
@@ -196,125 +121,82 @@ def fit_preprocess_operations(data, preprocessing_dict):
|
|
|
196
121
|
----------
|
|
197
122
|
data : pd.DataFrame
|
|
198
123
|
DataFrame to manipulate.
|
|
199
|
-
preprocessing_dict :
|
|
124
|
+
preprocessing_dict : PreprocessOpConfig
|
|
200
125
|
Dictionary of the operations and the features on which to operate.
|
|
201
126
|
|
|
202
127
|
Returns
|
|
203
128
|
-------
|
|
204
|
-
operation_dict :
|
|
129
|
+
operation_dict : Mapping[str, PreprocessOp | str]
|
|
205
130
|
Dictionary of the different preprocessing objects.
|
|
206
131
|
|
|
207
132
|
Raises
|
|
208
133
|
------
|
|
209
134
|
TypeError
|
|
210
135
|
If data is not a pd.DataFrame.
|
|
211
|
-
If features is not a list
|
|
136
|
+
If features is not a list[str].
|
|
212
137
|
KeyError
|
|
213
138
|
If a features is not a valid key.
|
|
214
139
|
ValueError
|
|
215
140
|
If preprocess is not a valid preprocessing function.
|
|
216
141
|
|
|
217
142
|
"""
|
|
218
|
-
if
|
|
219
|
-
raise TypeError(f"data should be a pd.DataFrame, but got {type(data)}")
|
|
143
|
+
if not isinstance(data, pd.DataFrame):
|
|
144
|
+
raise TypeError(f"data should be a pd.DataFrame, but got {type(data).__name__}")
|
|
145
|
+
|
|
146
|
+
# Use shallow copy instead of deepcopy for performance
|
|
147
|
+
data_working = data.copy()
|
|
148
|
+
operation_dict: dict[str, PreprocessOp | str] = {}
|
|
220
149
|
|
|
221
|
-
#
|
|
222
|
-
|
|
223
|
-
|
|
150
|
+
# Map names to classes for cleaner logic
|
|
151
|
+
transformers = {
|
|
152
|
+
"ordinal_encoder": OrdinalEncoder,
|
|
153
|
+
"standardise": StandardScaler,
|
|
154
|
+
"power_transform": PowerTransformer,
|
|
155
|
+
}
|
|
224
156
|
|
|
225
|
-
for
|
|
226
|
-
features =
|
|
157
|
+
for op_name, config in preprocessing_dict.items():
|
|
158
|
+
features = config.get("feature_list")
|
|
227
159
|
|
|
228
|
-
if
|
|
229
|
-
raise TypeError(
|
|
160
|
+
if not isinstance(features, list):
|
|
161
|
+
raise TypeError(
|
|
162
|
+
f"features should be a list, but got {type(features).__name__}"
|
|
163
|
+
)
|
|
230
164
|
|
|
231
|
-
if
|
|
165
|
+
if features and not isinstance(features[0], str):
|
|
232
166
|
raise TypeError(
|
|
233
|
-
f"features
|
|
167
|
+
f"features must contain strings, got {type(features[0]).__name__}"
|
|
234
168
|
)
|
|
235
169
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
data_copy[features] = operation_dict[preprocess].transform(
|
|
250
|
-
data_copy[features]
|
|
251
|
-
)
|
|
252
|
-
case "bin":
|
|
253
|
-
operation_dict[preprocess] = "bin"
|
|
254
|
-
case _:
|
|
255
|
-
raise ValueError(f"{preprocess} invalid preprocessing function")
|
|
170
|
+
if op_name == "bin":
|
|
171
|
+
operation_dict[op_name] = "bin"
|
|
172
|
+
continue
|
|
173
|
+
|
|
174
|
+
if op_name in transformers:
|
|
175
|
+
# Instantiate, fit, and transform sequentially
|
|
176
|
+
model = transformers[op_name]()
|
|
177
|
+
operation_dict[op_name] = model.fit(data_working[features])
|
|
178
|
+
|
|
179
|
+
# Update working data so subsequent fits see transformed values
|
|
180
|
+
data_working[features] = model.transform(data_working[features])
|
|
181
|
+
else:
|
|
182
|
+
raise ValueError(f"{op_name} is an invalid preprocessing function")
|
|
256
183
|
|
|
257
184
|
return operation_dict
|
|
258
185
|
|
|
259
186
|
|
|
260
|
-
def bin_score(data):
|
|
187
|
+
def bin_score(data: npt.NDArray) -> npt.NDArray:
|
|
261
188
|
"""
|
|
262
189
|
Bins the M3 score into 5 categories.
|
|
263
190
|
|
|
264
191
|
Parameters
|
|
265
192
|
----------
|
|
266
|
-
data :
|
|
193
|
+
data : npt.NDArray
|
|
267
194
|
M3 score data.
|
|
268
195
|
|
|
269
196
|
Returns
|
|
270
197
|
-------
|
|
271
|
-
binned_data :
|
|
198
|
+
binned_data : npt.NDArray
|
|
272
199
|
Binned data.
|
|
273
200
|
|
|
274
201
|
"""
|
|
275
|
-
|
|
276
|
-
binned_data[binned_data > 4] = 4
|
|
277
|
-
return binned_data
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
def extract_labels(data, labels):
|
|
281
|
-
"""
|
|
282
|
-
Extracts the prediction labels from the training data.
|
|
283
|
-
|
|
284
|
-
Parameters
|
|
285
|
-
----------
|
|
286
|
-
data : pd.DataFrame
|
|
287
|
-
DataFrame to manipulate.
|
|
288
|
-
labels : list(str)
|
|
289
|
-
List of labels to extract from the data.
|
|
290
|
-
|
|
291
|
-
Returns
|
|
292
|
-
-------
|
|
293
|
-
X : pd.DataFrame
|
|
294
|
-
DataFrame containing the data.
|
|
295
|
-
y : array-like
|
|
296
|
-
Array containing the prediction labels.
|
|
297
|
-
|
|
298
|
-
Raises
|
|
299
|
-
------
|
|
300
|
-
TypeError
|
|
301
|
-
If data is not a pd.DataFrame.
|
|
302
|
-
TypeError
|
|
303
|
-
If labels is not list(str).
|
|
304
|
-
KeyError
|
|
305
|
-
If a prediction label is not a valid key.
|
|
306
|
-
|
|
307
|
-
"""
|
|
308
|
-
if type(data) is not type(pd.DataFrame()):
|
|
309
|
-
raise TypeError(f"data should be a pd.DataFrame, but got {type(data)}")
|
|
310
|
-
|
|
311
|
-
if type(labels) is not type([]):
|
|
312
|
-
raise TypeError(f"labels should be a list, but got {type(labels)}")
|
|
313
|
-
|
|
314
|
-
if type(labels[0]) is not type(""):
|
|
315
|
-
raise TypeError(f"labels should be a list(str), but got {type(labels[0])}")
|
|
316
|
-
|
|
317
|
-
X = data.drop(labels, axis=1)
|
|
318
|
-
y = data[labels]
|
|
319
|
-
|
|
320
|
-
return X, y.to_numpy()
|
|
202
|
+
return np.clip(np.ceil(data), 0, 4).astype(int)
|