medpipe 0.1.2.dev0__py3-none-any.whl → 0.2.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
medpipe/__init__.py CHANGED
@@ -21,7 +21,7 @@ from .metrics import (
21
21
  print_metrics_CI,
22
22
  )
23
23
  from .models import get_full_proba, get_positive_proba, load_pipeline, save_pipeline
24
- from .pipeline.Pipeline import Pipeline
24
+ from .pipeline.pipeline import Pipeline
25
25
  from .utils import (
26
26
  exception_handler,
27
27
  load_data_from_csv,
@@ -52,3 +52,4 @@ __all__ = [
52
52
  "extract_data_from_db",
53
53
  "extract_labels",
54
54
  ]
55
+ __version__ = "v0.2.0.dev0"
medpipe/_types.py ADDED
@@ -0,0 +1,50 @@
1
+ """
2
+ Medpipe typings.
3
+
4
+ This module provides special types defined for the medpipe package.
5
+
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import TYPE_CHECKING, Annotated, Literal, Sequence, TypeAlias, TypeVar
11
+
12
+ import numpy as np
13
+ import numpy.typing as npt
14
+ import pandas as pd
15
+ from sklearn.ensemble import HistGradientBoostingClassifier
16
+ from sklearn.isotonic import IsotonicRegression
17
+ from sklearn.linear_model import LogisticRegression
18
+ from sklearn.preprocessing import OrdinalEncoder, PowerTransformer, StandardScaler
19
+
20
+ if TYPE_CHECKING:
21
+ from medpipe.models.calibrators import IsotonicCalibrator, LogisticCalibrator
22
+ from medpipe.models.predictors import HGBClassifier
23
+
24
+ # Define data and label types
25
+ SingleClassLabels: TypeAlias = Annotated[npt.NDArray[np.integer], Literal["N"]]
26
+ MultiClassLabels: TypeAlias = Annotated[npt.NDArray[np.integer], Literal["N", "C"]]
27
+ FullProba: TypeAlias = Annotated[npt.NDArray[np.number], Literal["C", "N", 2]]
28
+ PosProba: TypeAlias = Annotated[npt.NDArray[np.number], Literal["N", "C"]]
29
+ Labels: TypeAlias = SingleClassLabels | MultiClassLabels
30
+ PredData: TypeAlias = pd.DataFrame | npt.NDArray # Predictor class data
31
+ Data: TypeAlias = PredData | PosProba # Pipeline class data
32
+
33
+ # Define preprocessing types
34
+ PreprocessOpConfig: TypeAlias = dict[str, dict[str, list[str]]]
35
+ PreprocessOp: TypeAlias = OrdinalEncoder | StandardScaler | PowerTransformer
36
+
37
+ # Define metric types
38
+ CI: TypeAlias = tuple[npt.NDArray, npt.NDArray, npt.NDArray]
39
+ CIDict: TypeAlias = dict[str, CI]
40
+ MetricDict: TypeAlias = dict[str, list[float]]
41
+ ModelMetrics: TypeAlias = dict[str, dict[str, Sequence[float]]]
42
+
43
+ # Define model types
44
+ Classifier: TypeAlias = HistGradientBoostingClassifier
45
+ Predictor: TypeAlias = "HGBClassifier"
46
+ Regressor: TypeAlias = LogisticRegression | IsotonicRegression
47
+ Calibrator: TypeAlias = "IsotonicCalibrator | LogisticCalibrator"
48
+ Model: TypeAlias = Classifier | Regressor
49
+ R = TypeVar("R", bound=Regressor) # Generic Type Variable for Regressors
50
+ C = TypeVar("C", bound=Classifier) # Generic Type Variable for Classifiers
medpipe/data/__init__.py CHANGED
@@ -6,9 +6,10 @@ submodules:
6
6
  - preprocessing: contains functions for preprocessing data.
7
7
  - weighting: contains functions to compute sample weights for imbalanced data.
8
8
  - sampler: contains functions to sample the data for imbalanced data.
9
- - Preprocessor: class to prepare data for fitting.
9
+ - preprocessor: class to prepare data for fitting.
10
+ - utils: utility functions for data manipulation.
10
11
  """
11
12
 
12
- from . import Preprocessor, db, preprocessing, sampler, weighting
13
+ from . import db, preprocessing, preprocessor, sampler, utils, weighting
13
14
  from .db import extract_data_from_db
14
- from .preprocessing import extract_labels
15
+ from .utils import extract_labels
medpipe/data/db.py CHANGED
@@ -8,11 +8,13 @@ Functions:
8
8
  - extract_data_from_db: Queries a SQL .db to extract data.
9
9
  """
10
10
 
11
+ from __future__ import annotations
12
+
11
13
  import sqlite3
12
14
 
13
15
  import pandas as pd
14
16
 
15
- import medpipe.utils.exceptions
17
+ from medpipe.utils.exceptions import file_checks
16
18
 
17
19
 
18
20
  def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") -> None:
@@ -21,11 +23,11 @@ def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") ->
21
23
 
22
24
  Parameters
23
25
  ----------
24
- parquet_file
26
+ parquet_file : str
25
27
  File path to the .parquet file.
26
- db_file
28
+ db_file : str
27
29
  File path to the .db file.
28
- table_name : default: 'main'
30
+ table_name : str, default: 'main'
29
31
  Name of the table to create in the SQL database.
30
32
 
31
33
  Returns
@@ -48,12 +50,12 @@ def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") ->
48
50
 
49
51
  """
50
52
  try:
51
- medpipe.utils.exceptions.file_checks(parquet_file, ".parquet")
53
+ file_checks(parquet_file, ".parquet")
52
54
  except (FileNotFoundError, IsADirectoryError, TypeError, ValueError):
53
55
  raise
54
56
 
55
57
  try:
56
- medpipe.utils.exceptions.file_checks(db_file, ".db", exists=False)
58
+ file_checks(db_file, ".db", exists=False)
57
59
  except (FileNotFoundError, TypeError, ValueError, IsADirectoryError):
58
60
  raise
59
61
 
@@ -69,7 +71,7 @@ def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") ->
69
71
  conn.close()
70
72
 
71
73
 
72
- def extract_data_from_db(db_file: str, query: str):
74
+ def extract_data_from_db(db_file: str, query: str) -> pd.DataFrame:
73
75
  """
74
76
  Extracts data from a .db and saves it to a .csv file.
75
77
 
@@ -77,9 +79,9 @@ def extract_data_from_db(db_file: str, query: str):
77
79
 
78
80
  Parameters
79
81
  ----------
80
- db_file
82
+ db_file : str
81
83
  Path to the .db file.
82
- query
84
+ query : str
83
85
  Query to send to extract data.
84
86
 
85
87
  Returns
@@ -100,7 +102,7 @@ def extract_data_from_db(db_file: str, query: str):
100
102
 
101
103
  """
102
104
  try:
103
- medpipe.utils.exceptions.file_checks(db_file, ".db")
105
+ file_checks(db_file, ".db")
104
106
  except (FileNotFoundError, IsADirectoryError, TypeError, ValueError):
105
107
  raise
106
108
 
@@ -5,24 +5,29 @@ This module provides functions to preprocess data before training.
5
5
 
6
6
  Functions:
7
7
  - train_test_it: Creates a KFold iterator to split data into test and train sets.
8
- - get_validation_idx: Removes some of the indices to create a validation set.
9
8
  - convert_object_to_categorical: Converts object columns to categoricals.
10
9
  - fit_preprocess_operations: Fits processing operations to data.
11
10
  - bin_score: Bins the M3 score into 5 categories.
12
- - extract_labels: Extracts prediction labels from data.
13
11
  """
14
12
 
15
- from copy import deepcopy
13
+ from __future__ import annotations
14
+
15
+ from typing import TYPE_CHECKING, Any, Mapping
16
16
 
17
17
  import numpy as np
18
18
  import pandas as pd
19
- import sklearn as skl
19
+ from sklearn.model_selection import GroupKFold, StratifiedKFold
20
20
  from sklearn.preprocessing import OrdinalEncoder, PowerTransformer, StandardScaler
21
21
 
22
- from medpipe.utils.exceptions import array_check, array_dim_check
22
+ from medpipe._types import PreprocessOp, PreprocessOpConfig
23
+
24
+ if TYPE_CHECKING:
25
+ import numpy.typing as npt
23
26
 
24
27
 
25
- def train_test_it(group_k_fold=False, **kwargs):
28
+ def train_test_it(
29
+ group_k_fold: bool = False, **kwargs: Any
30
+ ) -> StratifiedKFold | GroupKFold:
26
31
  """
27
32
  Creates a KFold iterator to split data into test and train sets.
28
33
 
@@ -31,12 +36,12 @@ def train_test_it(group_k_fold=False, **kwargs):
31
36
  group_k_fold : bool, default: False
32
37
  If True, the data will be split using a group and a
33
38
  GroupKFold iterator is returned.
34
- **kwargs
39
+ **kwargs: Any
35
40
  Extra arguments for the StratifiedKFold or GroupKFold class.
36
41
 
37
42
  Returns
38
43
  -------
39
- kfold_it : StratifiedKFold or GroupKFold
44
+ kfold_it : StratifiedKFold | GroupKFold
40
45
  KFold iterator.
41
46
 
42
47
  Raises
@@ -66,103 +71,20 @@ def train_test_it(group_k_fold=False, **kwargs):
66
71
  args_dict.update({key: value})
67
72
 
68
73
  if not group_k_fold:
69
- kfold_it = skl.model_selection.StratifiedKFold(**args_dict)
74
+ kfold_it = StratifiedKFold(**args_dict)
70
75
  else:
71
- kfold_it = skl.model_selection.GroupKFold(**args_dict)
76
+ kfold_it = GroupKFold(**args_dict)
72
77
 
73
78
  return kfold_it
74
79
 
75
80
 
76
- def get_validation_idx(idx_list, groups=None, group_vals=None, val_size=0.1):
77
- """
78
- Removes some of the indices to create a validation set.
79
-
80
- If groups are provided and group_vals is None, all the indices of
81
- the group with the largest value are selected as the validation set.
82
- If group_vals is specified with groups, then the group_vals are
83
- selected as the validation/test set.
84
-
85
- Parameters
86
- ----------
87
- idx_list : np.array(n_samples,)
88
- Indices of the set to split.
89
- groups : pd.Series(n_samples,) or np.ndarray or None, default: None
90
- Groups to which the train indices belong.
91
- group_vals : list[] or None, default: None
92
- Group values that should be in the test set.
93
- val_size : float, default: 0.1
94
- Size of the validation set if groups are None.
95
-
96
- Returns
97
- -------
98
- train_idx : np.array
99
- Train indices.
100
- val_idx : np.array
101
- Validation indices.
102
-
103
- Raises
104
- ------
105
- TypeError
106
- If groups is not pd.Series or np.ndarray.
107
- If group_vals is not iterable.
108
- If group_vals is not a list or a np.ndarray.
109
- If val_size is not a float.
110
- ValueError
111
- If val_size < 0 or val_size > 1.
112
-
113
- """
114
- array_check(idx_list)
115
- if groups is not None:
116
- # If groups are provided
117
- if isinstance(groups, pd.Series):
118
- groups = groups.to_numpy() # Convert to array
119
- elif not isinstance(groups, np.ndarray):
120
- raise TypeError(
121
- f"groups should be a pd.Series or np.array, but got {type(groups)}"
122
- )
123
- array_check(idx_list)
124
- array_dim_check(idx_list, groups, dim=0)
125
-
126
- if group_vals is not None:
127
- if not hasattr(group_vals, "__iter__"):
128
- raise TypeError("group_vals should be iterable")
129
- if (
130
- isinstance(group_vals, dict)
131
- or isinstance(group_vals, str)
132
- or isinstance(group_vals, tuple)
133
- ):
134
- raise TypeError("group_vals should be list or array")
135
- val_idx = np.array([], dtype=np.int64)
136
- train_idx = []
137
- for group in group_vals:
138
- val_idx = np.concatenate((val_idx, np.where(groups == group)[0]))
139
-
140
- train_idx = np.setdiff1d(np.arange(len(groups)), val_idx)
141
-
142
- else:
143
- group_max = np.max(groups)
144
- val_idx = np.where(groups == group_max)[0]
145
- train_idx = np.where(groups != group_max)[0]
146
-
147
- else:
148
- if not isinstance(val_size, float):
149
- raise TypeError(f"val_size should be a float, but got {type(val_size)}")
150
- if val_size < 0.0 or val_size > 1.0:
151
- raise ValueError(f"val_size should be between 0 and 1, but got {val_size}")
152
- train_idx, val_idx = skl.model_selection.train_test_split(
153
- idx_list, test_size=val_size, random_state=42
154
- )
155
-
156
- return train_idx, val_idx
157
-
158
-
159
81
  def convert_object_to_categorical(data: pd.DataFrame) -> pd.DataFrame:
160
82
  """
161
83
  Converts all object columns of a DataFrame to categoricals.
162
84
 
163
85
  Parameters
164
86
  ----------
165
- data
87
+ data : pd.DataFrame
166
88
  DataFrame to manipulate.
167
89
 
168
90
  Returns
@@ -176,19 +98,22 @@ def convert_object_to_categorical(data: pd.DataFrame) -> pd.DataFrame:
176
98
  If data is not a pd.DataFrame.
177
99
 
178
100
  """
179
- if type(data) is not type(pd.DataFrame()):
180
- raise TypeError(f"data should be a pd.DataFrame, but got {type(data)}")
101
+ if not isinstance(data, pd.DataFrame):
102
+ raise TypeError(f"data should be a pd.DataFrame, but got {type(data).__name__}")
181
103
 
182
- # Create a copy of data to work on
183
- processed_data = data
104
+ processed_data = data.copy()
105
+ obj_cols = processed_data.select_dtypes(include=["object"]).columns
184
106
 
185
- for column in data.select_dtypes(include=["object"]).columns:
186
- processed_data[column] = data[column].astype("category")
107
+ # Vectorized conversion for all object columns at once
108
+ if not obj_cols.empty:
109
+ processed_data[obj_cols] = processed_data[obj_cols].astype("category")
187
110
 
188
111
  return processed_data
189
112
 
190
113
 
191
- def fit_preprocess_operations(data, preprocessing_dict):
114
+ def fit_preprocess_operations(
115
+ data: pd.DataFrame, preprocessing_dict: PreprocessOpConfig
116
+ ) -> Mapping[str, PreprocessOp | str]:
192
117
  """
193
118
  Fits processing operations to data.
194
119
 
@@ -196,125 +121,82 @@ def fit_preprocess_operations(data, preprocessing_dict):
196
121
  ----------
197
122
  data : pd.DataFrame
198
123
  DataFrame to manipulate.
199
- preprocessing_dict : dict[str, list[str]]
124
+ preprocessing_dict : PreprocessOpConfig
200
125
  Dictionary of the operations and the features on which to operate.
201
126
 
202
127
  Returns
203
128
  -------
204
- operation_dict : dict[]
129
+ operation_dict : Mapping[str, PreprocessOp | str]
205
130
  Dictionary of the different preprocessing objects.
206
131
 
207
132
  Raises
208
133
  ------
209
134
  TypeError
210
135
  If data is not a pd.DataFrame.
211
- If features is not a list(str).
136
+ If features is not a list[str].
212
137
  KeyError
213
138
  If a features is not a valid key.
214
139
  ValueError
215
140
  If preprocess is not a valid preprocessing function.
216
141
 
217
142
  """
218
- if type(data) is not type(pd.DataFrame()):
219
- raise TypeError(f"data should be a pd.DataFrame, but got {type(data)}")
143
+ if not isinstance(data, pd.DataFrame):
144
+ raise TypeError(f"data should be a pd.DataFrame, but got {type(data).__name__}")
145
+
146
+ # Use shallow copy instead of deepcopy for performance
147
+ data_working = data.copy()
148
+ operation_dict: dict[str, PreprocessOp | str] = {}
220
149
 
221
- # Operation dictionary to store fitted operations
222
- data_copy = deepcopy(data)
223
- operation_dict = dict()
150
+ # Map names to classes for cleaner logic
151
+ transformers = {
152
+ "ordinal_encoder": OrdinalEncoder,
153
+ "standardise": StandardScaler,
154
+ "power_transform": PowerTransformer,
155
+ }
224
156
 
225
- for preprocess in preprocessing_dict.keys():
226
- features = preprocessing_dict[preprocess]["feature_list"]
157
+ for op_name, config in preprocessing_dict.items():
158
+ features = config.get("feature_list")
227
159
 
228
- if type(features) is not type([]):
229
- raise TypeError(f"features should be a list, but got {type(features)}")
160
+ if not isinstance(features, list):
161
+ raise TypeError(
162
+ f"features should be a list, but got {type(features).__name__}"
163
+ )
230
164
 
231
- if type(features[0]) is not type(""):
165
+ if features and not isinstance(features[0], str):
232
166
  raise TypeError(
233
- f"features should be a list(str), but got list({type(features[0])}"
167
+ f"features must contain strings, got {type(features[0]).__name__}"
234
168
  )
235
169
 
236
- match preprocess:
237
- case "ordinal_encoder":
238
- operation_dict[preprocess] = OrdinalEncoder().fit(data_copy[features])
239
- data_copy[features] = operation_dict[preprocess].transform(
240
- data_copy[features]
241
- )
242
- case "standardise":
243
- operation_dict[preprocess] = StandardScaler().fit(data_copy[features])
244
- data_copy[features] = operation_dict[preprocess].transform(
245
- data_copy[features]
246
- )
247
- case "power_transform":
248
- operation_dict[preprocess] = PowerTransformer().fit(data_copy[features])
249
- data_copy[features] = operation_dict[preprocess].transform(
250
- data_copy[features]
251
- )
252
- case "bin":
253
- operation_dict[preprocess] = "bin"
254
- case _:
255
- raise ValueError(f"{preprocess} invalid preprocessing function")
170
+ if op_name == "bin":
171
+ operation_dict[op_name] = "bin"
172
+ continue
173
+
174
+ if op_name in transformers:
175
+ # Instantiate, fit, and transform sequentially
176
+ model = transformers[op_name]()
177
+ operation_dict[op_name] = model.fit(data_working[features])
178
+
179
+ # Update working data so subsequent fits see transformed values
180
+ data_working[features] = model.transform(data_working[features])
181
+ else:
182
+ raise ValueError(f"{op_name} is an invalid preprocessing function")
256
183
 
257
184
  return operation_dict
258
185
 
259
186
 
260
- def bin_score(data):
187
+ def bin_score(data: npt.NDArray) -> npt.NDArray:
261
188
  """
262
189
  Bins the M3 score into 5 categories.
263
190
 
264
191
  Parameters
265
192
  ----------
266
- data : pd.DataFrame
193
+ data : npt.NDArray
267
194
  M3 score data.
268
195
 
269
196
  Returns
270
197
  -------
271
- binned_data : pd.DataFrame
198
+ binned_data : npt.NDArray
272
199
  Binned data.
273
200
 
274
201
  """
275
- binned_data = np.ceil(data)
276
- binned_data[binned_data > 4] = 4
277
- return binned_data
278
-
279
-
280
- def extract_labels(data, labels):
281
- """
282
- Extracts the prediction labels from the training data.
283
-
284
- Parameters
285
- ----------
286
- data : pd.DataFrame
287
- DataFrame to manipulate.
288
- labels : list(str)
289
- List of labels to extract from the data.
290
-
291
- Returns
292
- -------
293
- X : pd.DataFrame
294
- DataFrame containing the data.
295
- y : array-like
296
- Array containing the prediction labels.
297
-
298
- Raises
299
- ------
300
- TypeError
301
- If data is not a pd.DataFrame.
302
- TypeError
303
- If labels is not list(str).
304
- KeyError
305
- If a prediction label is not a valid key.
306
-
307
- """
308
- if type(data) is not type(pd.DataFrame()):
309
- raise TypeError(f"data should be a pd.DataFrame, but got {type(data)}")
310
-
311
- if type(labels) is not type([]):
312
- raise TypeError(f"labels should be a list, but got {type(labels)}")
313
-
314
- if type(labels[0]) is not type(""):
315
- raise TypeError(f"labels should be a list(str), but got {type(labels[0])}")
316
-
317
- X = data.drop(labels, axis=1)
318
- y = data[labels]
319
-
320
- return X, y.to_numpy()
202
+ return np.clip(np.ceil(data), 0, 4).astype(int)