medpipe 0.1.2__tar.gz → 0.2.0.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/PKG-INFO +7 -2
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/README.md +6 -1
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/pyproject.toml +1 -1
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/__init__.py +2 -1
- medpipe-0.2.0.dev0/src/medpipe/_types.py +50 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/data/__init__.py +4 -3
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/data/db.py +12 -10
- medpipe-0.2.0.dev0/src/medpipe/data/preprocessing.py +202 -0
- medpipe-0.1.2/src/medpipe/data/Preprocessor.py → medpipe-0.2.0.dev0/src/medpipe/data/preprocessor.py +65 -53
- medpipe-0.2.0.dev0/src/medpipe/data/sampler.py +710 -0
- medpipe-0.2.0.dev0/src/medpipe/data/utils.py +269 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/data/weighting.py +77 -67
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/metrics/core.py +106 -138
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/metrics/plots.py +54 -33
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/models/__init__.py +1 -1
- medpipe-0.2.0.dev0/src/medpipe/models/calibrators.py +345 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/models/core.py +75 -43
- medpipe-0.2.0.dev0/src/medpipe/models/predictors.py +280 -0
- medpipe-0.2.0.dev0/src/medpipe/pipeline/__init__.py +8 -0
- medpipe-0.2.0.dev0/src/medpipe/pipeline/pipeline.py +822 -0
- medpipe-0.2.0.dev0/src/medpipe/py.typed +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/utils/config.py +18 -11
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/utils/exceptions.py +23 -12
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/utils/io.py +11 -5
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/utils/logger.py +13 -6
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe.egg-info/PKG-INFO +7 -2
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe.egg-info/SOURCES.txt +12 -7
- medpipe-0.2.0.dev0/test/test_data_preprocessing.py +132 -0
- medpipe-0.1.2/test/test_data_Preprocessor.py → medpipe-0.2.0.dev0/test/test_data_preprocessor.py +9 -4
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/test/test_data_sampler.py +209 -104
- medpipe-0.1.2/test/test_data_preprocessing.py → medpipe-0.2.0.dev0/test/test_data_utils.py +94 -112
- medpipe-0.2.0.dev0/test/test_models_calibrators.py +38 -0
- medpipe-0.2.0.dev0/test/test_models_predictors.py +34 -0
- medpipe-0.1.2/test/test_pipeline_Pipeline.py → medpipe-0.2.0.dev0/test/test_pipeline_pipeline.py +1 -1
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/test/test_utils_exceptions.py +4 -2
- medpipe-0.1.2/src/medpipe/data/preprocessing.py +0 -320
- medpipe-0.1.2/src/medpipe/data/sampler.py +0 -561
- medpipe-0.1.2/src/medpipe/models/Calibrator.py +0 -147
- medpipe-0.1.2/src/medpipe/models/Predictor.py +0 -144
- medpipe-0.1.2/src/medpipe/pipeline/Pipeline.py +0 -832
- medpipe-0.1.2/src/medpipe/pipeline/__init__.py +0 -8
- medpipe-0.1.2/test/test_metrics_core.py +0 -44
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/LICENSE +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/requirements.txt +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/setup.cfg +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/metrics/__init__.py +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe/utils/__init__.py +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe.egg-info/dependency_links.txt +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe.egg-info/requires.txt +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/src/medpipe.egg-info/top_level.txt +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/test/test_data_db.py +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/test/test_data_weighting.py +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/test/test_models_core.py +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/test/test_utils_config.py +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/test/test_utils_io.py +0 -0
- {medpipe-0.1.2 → medpipe-0.2.0.dev0}/test/test_utils_logger.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: medpipe
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0.dev0
|
|
4
4
|
Summary: A package for clinical AI pipeline development
|
|
5
5
|
Author-email: Mathias Roesler <mathias.roesler@auckland.co.nz>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -45,7 +45,12 @@ The **medpipe** package is a layer to help create AI models for clinical applica
|
|
|
45
45
|
___
|
|
46
46
|
## Installation
|
|
47
47
|
|
|
48
|
-
To install **medpipe**
|
|
48
|
+
To install **medpipe** use pip:
|
|
49
|
+
```
|
|
50
|
+
$ pip install medpipe
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
or clone the GitHub repository and install the package with pip:
|
|
49
54
|
```
|
|
50
55
|
$ git clone git@github.com:Surgical-Recovery-and-Safety-Lab/medpipe.git
|
|
51
56
|
$ cd medpipe
|
|
@@ -19,7 +19,12 @@ The **medpipe** package is a layer to help create AI models for clinical applica
|
|
|
19
19
|
___
|
|
20
20
|
## Installation
|
|
21
21
|
|
|
22
|
-
To install **medpipe**
|
|
22
|
+
To install **medpipe** use pip:
|
|
23
|
+
```
|
|
24
|
+
$ pip install medpipe
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
or clone the GitHub repository and install the package with pip:
|
|
23
28
|
```
|
|
24
29
|
$ git clone git@github.com:Surgical-Recovery-and-Safety-Lab/medpipe.git
|
|
25
30
|
$ cd medpipe
|
|
@@ -21,7 +21,7 @@ from .metrics import (
|
|
|
21
21
|
print_metrics_CI,
|
|
22
22
|
)
|
|
23
23
|
from .models import get_full_proba, get_positive_proba, load_pipeline, save_pipeline
|
|
24
|
-
from .pipeline.
|
|
24
|
+
from .pipeline.pipeline import Pipeline
|
|
25
25
|
from .utils import (
|
|
26
26
|
exception_handler,
|
|
27
27
|
load_data_from_csv,
|
|
@@ -52,3 +52,4 @@ __all__ = [
|
|
|
52
52
|
"extract_data_from_db",
|
|
53
53
|
"extract_labels",
|
|
54
54
|
]
|
|
55
|
+
__version__ = "v0.2.0.dev0"
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Medpipe typings.
|
|
3
|
+
|
|
4
|
+
This module provides special types defined for the medpipe package.
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import TYPE_CHECKING, Annotated, Literal, Sequence, TypeAlias, TypeVar
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
import numpy.typing as npt
|
|
14
|
+
import pandas as pd
|
|
15
|
+
from sklearn.ensemble import HistGradientBoostingClassifier
|
|
16
|
+
from sklearn.isotonic import IsotonicRegression
|
|
17
|
+
from sklearn.linear_model import LogisticRegression
|
|
18
|
+
from sklearn.preprocessing import OrdinalEncoder, PowerTransformer, StandardScaler
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from medpipe.models.calibrators import IsotonicCalibrator, LogisticCalibrator
|
|
22
|
+
from medpipe.models.predictors import HGBClassifier
|
|
23
|
+
|
|
24
|
+
# Define data and label types
|
|
25
|
+
SingleClassLabels: TypeAlias = Annotated[npt.NDArray[np.integer], Literal["N"]]
|
|
26
|
+
MultiClassLabels: TypeAlias = Annotated[npt.NDArray[np.integer], Literal["N", "C"]]
|
|
27
|
+
FullProba: TypeAlias = Annotated[npt.NDArray[np.number], Literal["C", "N", 2]]
|
|
28
|
+
PosProba: TypeAlias = Annotated[npt.NDArray[np.number], Literal["N", "C"]]
|
|
29
|
+
Labels: TypeAlias = SingleClassLabels | MultiClassLabels
|
|
30
|
+
PredData: TypeAlias = pd.DataFrame | npt.NDArray # Predictor class data
|
|
31
|
+
Data: TypeAlias = PredData | PosProba # Pipeline class data
|
|
32
|
+
|
|
33
|
+
# Define preprocessing types
|
|
34
|
+
PreprocessOpConfig: TypeAlias = dict[str, dict[str, list[str]]]
|
|
35
|
+
PreprocessOp: TypeAlias = OrdinalEncoder | StandardScaler | PowerTransformer
|
|
36
|
+
|
|
37
|
+
# Define metric types
|
|
38
|
+
CI: TypeAlias = tuple[npt.NDArray, npt.NDArray, npt.NDArray]
|
|
39
|
+
CIDict: TypeAlias = dict[str, CI]
|
|
40
|
+
MetricDict: TypeAlias = dict[str, list[float]]
|
|
41
|
+
ModelMetrics: TypeAlias = dict[str, dict[str, Sequence[float]]]
|
|
42
|
+
|
|
43
|
+
# Define model types
|
|
44
|
+
Classifier: TypeAlias = HistGradientBoostingClassifier
|
|
45
|
+
Predictor: TypeAlias = "HGBClassifier"
|
|
46
|
+
Regressor: TypeAlias = LogisticRegression | IsotonicRegression
|
|
47
|
+
Calibrator: TypeAlias = "IsotonicCalibrator | LogisticCalibrator"
|
|
48
|
+
Model: TypeAlias = Classifier | Regressor
|
|
49
|
+
R = TypeVar("R", bound=Regressor) # Generic Type Variable for Regressors
|
|
50
|
+
C = TypeVar("C", bound=Classifier) # Generic Type Variable for Classifiers
|
|
@@ -6,9 +6,10 @@ submodules:
|
|
|
6
6
|
- preprocessing: contains functions for preprocessing data.
|
|
7
7
|
- weighting: contains functions to compute sample weights for imbalanced data.
|
|
8
8
|
- sampler: contains functions to sample the data for imbalanced data.
|
|
9
|
-
-
|
|
9
|
+
- preprocessor: class to prepare data for fitting.
|
|
10
|
+
- utils: utility functions for data manipulation.
|
|
10
11
|
"""
|
|
11
12
|
|
|
12
|
-
from . import
|
|
13
|
+
from . import db, preprocessing, preprocessor, sampler, utils, weighting
|
|
13
14
|
from .db import extract_data_from_db
|
|
14
|
-
from .
|
|
15
|
+
from .utils import extract_labels
|
|
@@ -8,11 +8,13 @@ Functions:
|
|
|
8
8
|
- extract_data_from_db: Queries a SQL .db to extract data.
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
11
13
|
import sqlite3
|
|
12
14
|
|
|
13
15
|
import pandas as pd
|
|
14
16
|
|
|
15
|
-
|
|
17
|
+
from medpipe.utils.exceptions import file_checks
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") -> None:
|
|
@@ -21,11 +23,11 @@ def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") ->
|
|
|
21
23
|
|
|
22
24
|
Parameters
|
|
23
25
|
----------
|
|
24
|
-
parquet_file
|
|
26
|
+
parquet_file : str
|
|
25
27
|
File path to the .parquet file.
|
|
26
|
-
db_file
|
|
28
|
+
db_file : str
|
|
27
29
|
File path to the .db file.
|
|
28
|
-
table_name : default: 'main'
|
|
30
|
+
table_name : str, default: 'main'
|
|
29
31
|
Name of the table to create in the SQL database.
|
|
30
32
|
|
|
31
33
|
Returns
|
|
@@ -48,12 +50,12 @@ def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") ->
|
|
|
48
50
|
|
|
49
51
|
"""
|
|
50
52
|
try:
|
|
51
|
-
|
|
53
|
+
file_checks(parquet_file, ".parquet")
|
|
52
54
|
except (FileNotFoundError, IsADirectoryError, TypeError, ValueError):
|
|
53
55
|
raise
|
|
54
56
|
|
|
55
57
|
try:
|
|
56
|
-
|
|
58
|
+
file_checks(db_file, ".db", exists=False)
|
|
57
59
|
except (FileNotFoundError, TypeError, ValueError, IsADirectoryError):
|
|
58
60
|
raise
|
|
59
61
|
|
|
@@ -69,7 +71,7 @@ def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") ->
|
|
|
69
71
|
conn.close()
|
|
70
72
|
|
|
71
73
|
|
|
72
|
-
def extract_data_from_db(db_file: str, query: str):
|
|
74
|
+
def extract_data_from_db(db_file: str, query: str) -> pd.DataFrame:
|
|
73
75
|
"""
|
|
74
76
|
Extracts data from a .db and saves it to a .csv file.
|
|
75
77
|
|
|
@@ -77,9 +79,9 @@ def extract_data_from_db(db_file: str, query: str):
|
|
|
77
79
|
|
|
78
80
|
Parameters
|
|
79
81
|
----------
|
|
80
|
-
db_file
|
|
82
|
+
db_file : str
|
|
81
83
|
Path to the .db file.
|
|
82
|
-
query
|
|
84
|
+
query : str
|
|
83
85
|
Query to send to extract data.
|
|
84
86
|
|
|
85
87
|
Returns
|
|
@@ -100,7 +102,7 @@ def extract_data_from_db(db_file: str, query: str):
|
|
|
100
102
|
|
|
101
103
|
"""
|
|
102
104
|
try:
|
|
103
|
-
|
|
105
|
+
file_checks(db_file, ".db")
|
|
104
106
|
except (FileNotFoundError, IsADirectoryError, TypeError, ValueError):
|
|
105
107
|
raise
|
|
106
108
|
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Preprocessing functions module.
|
|
3
|
+
|
|
4
|
+
This module provides functions to preprocess data before training.
|
|
5
|
+
|
|
6
|
+
Functions:
|
|
7
|
+
- train_test_it: Creates a KFold iterator to split data into test and train sets.
|
|
8
|
+
- convert_object_to_categorical: Converts object columns to categoricals.
|
|
9
|
+
- fit_preprocess_operations: Fits processing operations to data.
|
|
10
|
+
- bin_score: Bins the M3 score into 5 categories.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from typing import TYPE_CHECKING, Any, Mapping
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
import pandas as pd
|
|
19
|
+
from sklearn.model_selection import GroupKFold, StratifiedKFold
|
|
20
|
+
from sklearn.preprocessing import OrdinalEncoder, PowerTransformer, StandardScaler
|
|
21
|
+
|
|
22
|
+
from medpipe._types import PreprocessOp, PreprocessOpConfig
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
import numpy.typing as npt
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def train_test_it(
|
|
29
|
+
group_k_fold: bool = False, **kwargs: Any
|
|
30
|
+
) -> StratifiedKFold | GroupKFold:
|
|
31
|
+
"""
|
|
32
|
+
Creates a KFold iterator to split data into test and train sets.
|
|
33
|
+
|
|
34
|
+
Parameters
|
|
35
|
+
----------
|
|
36
|
+
group_k_fold : bool, default: False
|
|
37
|
+
If True, the data will be split using a group and a
|
|
38
|
+
GroupKFold iterator is returned.
|
|
39
|
+
**kwargs: Any
|
|
40
|
+
Extra arguments for the StratifiedKFold or GroupKFold class.
|
|
41
|
+
|
|
42
|
+
Returns
|
|
43
|
+
-------
|
|
44
|
+
kfold_it : StratifiedKFold | GroupKFold
|
|
45
|
+
KFold iterator.
|
|
46
|
+
|
|
47
|
+
Raises
|
|
48
|
+
------
|
|
49
|
+
ValueError
|
|
50
|
+
If n_splits is less than 2.
|
|
51
|
+
|
|
52
|
+
"""
|
|
53
|
+
# Create the correct argument dict for StratifiedKFold
|
|
54
|
+
args_dict = dict()
|
|
55
|
+
for key, value in kwargs.items():
|
|
56
|
+
match key:
|
|
57
|
+
case "random_state":
|
|
58
|
+
if value == -1:
|
|
59
|
+
value = None
|
|
60
|
+
args_dict.update({key: value})
|
|
61
|
+
|
|
62
|
+
case "shuffle":
|
|
63
|
+
args_dict.update({key: value})
|
|
64
|
+
|
|
65
|
+
case "n_splits":
|
|
66
|
+
if value < 2:
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f"n_splits should be greater than 2, but got {value}"
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
args_dict.update({key: value})
|
|
72
|
+
|
|
73
|
+
if not group_k_fold:
|
|
74
|
+
kfold_it = StratifiedKFold(**args_dict)
|
|
75
|
+
else:
|
|
76
|
+
kfold_it = GroupKFold(**args_dict)
|
|
77
|
+
|
|
78
|
+
return kfold_it
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def convert_object_to_categorical(data: pd.DataFrame) -> pd.DataFrame:
|
|
82
|
+
"""
|
|
83
|
+
Converts all object columns of a DataFrame to categoricals.
|
|
84
|
+
|
|
85
|
+
Parameters
|
|
86
|
+
----------
|
|
87
|
+
data : pd.DataFrame
|
|
88
|
+
DataFrame to manipulate.
|
|
89
|
+
|
|
90
|
+
Returns
|
|
91
|
+
-------
|
|
92
|
+
processed_data : pd.DataFrame
|
|
93
|
+
Processed DataFrame.
|
|
94
|
+
|
|
95
|
+
Raises
|
|
96
|
+
------
|
|
97
|
+
TypeError
|
|
98
|
+
If data is not a pd.DataFrame.
|
|
99
|
+
|
|
100
|
+
"""
|
|
101
|
+
if not isinstance(data, pd.DataFrame):
|
|
102
|
+
raise TypeError(f"data should be a pd.DataFrame, but got {type(data).__name__}")
|
|
103
|
+
|
|
104
|
+
processed_data = data.copy()
|
|
105
|
+
obj_cols = processed_data.select_dtypes(include=["object"]).columns
|
|
106
|
+
|
|
107
|
+
# Vectorized conversion for all object columns at once
|
|
108
|
+
if not obj_cols.empty:
|
|
109
|
+
processed_data[obj_cols] = processed_data[obj_cols].astype("category")
|
|
110
|
+
|
|
111
|
+
return processed_data
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def fit_preprocess_operations(
|
|
115
|
+
data: pd.DataFrame, preprocessing_dict: PreprocessOpConfig
|
|
116
|
+
) -> Mapping[str, PreprocessOp | str]:
|
|
117
|
+
"""
|
|
118
|
+
Fits processing operations to data.
|
|
119
|
+
|
|
120
|
+
Parameters
|
|
121
|
+
----------
|
|
122
|
+
data : pd.DataFrame
|
|
123
|
+
DataFrame to manipulate.
|
|
124
|
+
preprocessing_dict : PreprocessOpConfig
|
|
125
|
+
Dictionary of the operations and the features on which to operate.
|
|
126
|
+
|
|
127
|
+
Returns
|
|
128
|
+
-------
|
|
129
|
+
operation_dict : Mapping[str, PreprocessOp | str]
|
|
130
|
+
Dictionary of the different preprocessing objects.
|
|
131
|
+
|
|
132
|
+
Raises
|
|
133
|
+
------
|
|
134
|
+
TypeError
|
|
135
|
+
If data is not a pd.DataFrame.
|
|
136
|
+
If features is not a list[str].
|
|
137
|
+
KeyError
|
|
138
|
+
If a features is not a valid key.
|
|
139
|
+
ValueError
|
|
140
|
+
If preprocess is not a valid preprocessing function.
|
|
141
|
+
|
|
142
|
+
"""
|
|
143
|
+
if not isinstance(data, pd.DataFrame):
|
|
144
|
+
raise TypeError(f"data should be a pd.DataFrame, but got {type(data).__name__}")
|
|
145
|
+
|
|
146
|
+
# Use shallow copy instead of deepcopy for performance
|
|
147
|
+
data_working = data.copy()
|
|
148
|
+
operation_dict: dict[str, PreprocessOp | str] = {}
|
|
149
|
+
|
|
150
|
+
# Map names to classes for cleaner logic
|
|
151
|
+
transformers = {
|
|
152
|
+
"ordinal_encoder": OrdinalEncoder,
|
|
153
|
+
"standardise": StandardScaler,
|
|
154
|
+
"power_transform": PowerTransformer,
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
for op_name, config in preprocessing_dict.items():
|
|
158
|
+
features = config.get("feature_list")
|
|
159
|
+
|
|
160
|
+
if not isinstance(features, list):
|
|
161
|
+
raise TypeError(
|
|
162
|
+
f"features should be a list, but got {type(features).__name__}"
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
if features and not isinstance(features[0], str):
|
|
166
|
+
raise TypeError(
|
|
167
|
+
f"features must contain strings, got {type(features[0]).__name__}"
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
if op_name == "bin":
|
|
171
|
+
operation_dict[op_name] = "bin"
|
|
172
|
+
continue
|
|
173
|
+
|
|
174
|
+
if op_name in transformers:
|
|
175
|
+
# Instantiate, fit, and transform sequentially
|
|
176
|
+
model = transformers[op_name]()
|
|
177
|
+
operation_dict[op_name] = model.fit(data_working[features])
|
|
178
|
+
|
|
179
|
+
# Update working data so subsequent fits see transformed values
|
|
180
|
+
data_working[features] = model.transform(data_working[features])
|
|
181
|
+
else:
|
|
182
|
+
raise ValueError(f"{op_name} is an invalid preprocessing function")
|
|
183
|
+
|
|
184
|
+
return operation_dict
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def bin_score(data: npt.NDArray) -> npt.NDArray:
|
|
188
|
+
"""
|
|
189
|
+
Bins the M3 score into 5 categories.
|
|
190
|
+
|
|
191
|
+
Parameters
|
|
192
|
+
----------
|
|
193
|
+
data : npt.NDArray
|
|
194
|
+
M3 score data.
|
|
195
|
+
|
|
196
|
+
Returns
|
|
197
|
+
-------
|
|
198
|
+
binned_data : npt.NDArray
|
|
199
|
+
Binned data.
|
|
200
|
+
|
|
201
|
+
"""
|
|
202
|
+
return np.clip(np.ceil(data), 0, 4).astype(int)
|
medpipe-0.1.2/src/medpipe/data/Preprocessor.py → medpipe-0.2.0.dev0/src/medpipe/data/preprocessor.py
RENAMED
|
@@ -5,8 +5,13 @@ This class creates a Preprocessor to prepare data.
|
|
|
5
5
|
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
from
|
|
8
|
+
from __future__ import annotations
|
|
9
9
|
|
|
10
|
+
from typing import TYPE_CHECKING, Mapping
|
|
11
|
+
|
|
12
|
+
import pandas as pd
|
|
13
|
+
|
|
14
|
+
from medpipe._types import PreprocessOp, PreprocessOpConfig
|
|
10
15
|
from medpipe.utils.logger import print_message
|
|
11
16
|
|
|
12
17
|
from .preprocessing import (
|
|
@@ -14,8 +19,14 @@ from .preprocessing import (
|
|
|
14
19
|
convert_object_to_categorical,
|
|
15
20
|
fit_preprocess_operations,
|
|
16
21
|
)
|
|
22
|
+
from .utils import downcast_dtypes
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
import logging
|
|
26
|
+
|
|
27
|
+
import pandas as pd
|
|
17
28
|
|
|
18
|
-
SCRIPT_NAME = "data/
|
|
29
|
+
SCRIPT_NAME = "data/preprocessor"
|
|
19
30
|
|
|
20
31
|
|
|
21
32
|
class Preprocessor:
|
|
@@ -26,9 +37,11 @@ class Preprocessor:
|
|
|
26
37
|
----------
|
|
27
38
|
preprocess : bool
|
|
28
39
|
Flag to preprocess data or not.
|
|
29
|
-
transform_seq :
|
|
40
|
+
transform_seq : PreprocessOpConfig
|
|
30
41
|
Transformation sequence for the data.
|
|
31
|
-
|
|
42
|
+
operations : Mapping[str, PreprocessorOp | str]
|
|
43
|
+
Preprocessing operations that are fitted.
|
|
44
|
+
logger : logging.Logger | None, default: None
|
|
32
45
|
Logger object to log prints. If None print to terminal.
|
|
33
46
|
|
|
34
47
|
Methods
|
|
@@ -45,15 +58,19 @@ class Preprocessor:
|
|
|
45
58
|
Transforms input data based on fitted operations.
|
|
46
59
|
"""
|
|
47
60
|
|
|
48
|
-
def __init__(
|
|
61
|
+
def __init__(
|
|
62
|
+
self,
|
|
63
|
+
preprocessor_config: PreprocessOpConfig,
|
|
64
|
+
logger: logging.Logger | None = None,
|
|
65
|
+
) -> None:
|
|
49
66
|
"""
|
|
50
67
|
Initialise a Preprocessor class instance.
|
|
51
68
|
|
|
52
69
|
Parameters
|
|
53
70
|
----------
|
|
54
|
-
preprocessor_config :
|
|
71
|
+
preprocessor_config : PreprocessorOpConfig
|
|
55
72
|
Configuration parameters for the preprocessor object.
|
|
56
|
-
logger : logging.Logger
|
|
73
|
+
logger : logging.Logger | None, default: None
|
|
57
74
|
Logger object to log prints. If None print to terminal.
|
|
58
75
|
|
|
59
76
|
Returns
|
|
@@ -64,10 +81,12 @@ class Preprocessor:
|
|
|
64
81
|
"""
|
|
65
82
|
self.preprocess = preprocessor_config.pop("preprocess")
|
|
66
83
|
self.transform_seq = preprocessor_config
|
|
67
|
-
self.operations
|
|
84
|
+
self.operations: Mapping[str, PreprocessOp | str] = (
|
|
85
|
+
dict()
|
|
86
|
+
) # Empty dict to contain operations
|
|
68
87
|
self.logger = logger
|
|
69
88
|
|
|
70
|
-
def _clean_data(self, X):
|
|
89
|
+
def _clean_data(self, X: pd.DataFrame) -> pd.DataFrame:
|
|
71
90
|
"""
|
|
72
91
|
Cleans data before transformation.
|
|
73
92
|
|
|
@@ -76,21 +95,22 @@ class Preprocessor:
|
|
|
76
95
|
|
|
77
96
|
Parameters
|
|
78
97
|
----------
|
|
79
|
-
X : pd.Dataframe
|
|
80
|
-
Data to clean.
|
|
98
|
+
X : pd.Dataframe
|
|
99
|
+
Data to clean of shape (n_samples, n_features).
|
|
81
100
|
|
|
82
101
|
Returns
|
|
83
102
|
-------
|
|
84
|
-
data : pd.Dataframe
|
|
85
|
-
Cleaned data.
|
|
103
|
+
data : pd.Dataframe
|
|
104
|
+
Cleaned data of shape (n_samples, n_features).
|
|
86
105
|
|
|
87
106
|
"""
|
|
88
107
|
# Convert objects to categorical (not saved so needs to be here)
|
|
89
108
|
data = convert_object_to_categorical(X)
|
|
90
109
|
|
|
91
|
-
# Remove NaN values
|
|
92
110
|
nb_nan_rows = data.isna().any(axis=1).sum()
|
|
93
|
-
|
|
111
|
+
|
|
112
|
+
if nb_nan_rows > 0:
|
|
113
|
+
data = data.dropna()
|
|
94
114
|
|
|
95
115
|
print_message(
|
|
96
116
|
f"Dropped {nb_nan_rows} rows with NaN values", self.logger, SCRIPT_NAME
|
|
@@ -98,39 +118,32 @@ class Preprocessor:
|
|
|
98
118
|
|
|
99
119
|
return data
|
|
100
120
|
|
|
101
|
-
def fit_transform(self, X):
|
|
121
|
+
def fit_transform(self, X: pd.DataFrame) -> pd.DataFrame:
|
|
102
122
|
"""
|
|
103
123
|
Fits the operations and transforms the input data.
|
|
104
124
|
|
|
105
125
|
Parameters
|
|
106
126
|
----------
|
|
107
|
-
X : pd.Dataframe
|
|
108
|
-
Data to clean.
|
|
127
|
+
X : pd.Dataframe
|
|
128
|
+
Data of shape (n_samples, n_features) to clean.
|
|
109
129
|
|
|
110
130
|
Returns
|
|
111
131
|
-------
|
|
112
|
-
data : pd.Dataframe
|
|
113
|
-
Transformed data.
|
|
132
|
+
data : pd.Dataframe
|
|
133
|
+
Transformed data of shape (n_samples, n_features).
|
|
114
134
|
|
|
115
135
|
"""
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
if self.preprocess:
|
|
119
|
-
# If the preprocess flag is true
|
|
120
|
-
print_message("Fitting processing operations", self.logger, SCRIPT_NAME)
|
|
121
|
-
self.operations = fit_preprocess_operations(data, self.transform_seq)
|
|
122
|
-
data = self.transform(data)
|
|
136
|
+
self.fit(X) # Fit operations
|
|
137
|
+
return self.transform(X) # Transform data
|
|
123
138
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
def fit(self, X):
|
|
139
|
+
def fit(self, X: pd.DataFrame) -> None:
|
|
127
140
|
"""
|
|
128
141
|
Fits the operations based on input data.
|
|
129
142
|
|
|
130
143
|
Parameters
|
|
131
144
|
----------
|
|
132
|
-
X : pd.Dataframe
|
|
133
|
-
Data to clean.
|
|
145
|
+
X : pd.Dataframe
|
|
146
|
+
Data to clean of shape (n_samples, n_features).
|
|
134
147
|
|
|
135
148
|
Returns
|
|
136
149
|
-------
|
|
@@ -138,42 +151,41 @@ class Preprocessor:
|
|
|
138
151
|
Nothings is returned.
|
|
139
152
|
|
|
140
153
|
"""
|
|
141
|
-
|
|
154
|
+
if not self.preprocess: # Return None if preprocess flag is False
|
|
155
|
+
return
|
|
142
156
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
self.operations = fit_preprocess_operations(data, self.transform_seq)
|
|
157
|
+
data = self._clean_data(X) # Clean data before transformation
|
|
158
|
+
print_message("Fitting preprocessing operations", self.logger, SCRIPT_NAME)
|
|
159
|
+
self.operations = fit_preprocess_operations(data, self.transform_seq)
|
|
147
160
|
|
|
148
|
-
def transform(self, X):
|
|
161
|
+
def transform(self, X: pd.DataFrame) -> pd.DataFrame:
|
|
149
162
|
"""
|
|
150
163
|
Transforms input data based on fitted operations.
|
|
151
164
|
|
|
152
165
|
Parameters
|
|
153
166
|
----------
|
|
154
|
-
X : pd.Dataframe
|
|
155
|
-
Data to clean.
|
|
167
|
+
X : pd.Dataframe
|
|
168
|
+
Data to clean of shape (n_samples, n_features).
|
|
156
169
|
|
|
157
170
|
Returns
|
|
158
171
|
-------
|
|
159
|
-
data : pd.Dataframe
|
|
160
|
-
Transformed data.
|
|
172
|
+
data : pd.Dataframe
|
|
173
|
+
Transformed data of shape (n_samples, n_features).
|
|
161
174
|
|
|
162
175
|
"""
|
|
163
|
-
data = self._clean_data(
|
|
176
|
+
data = self._clean_data(X.copy()) # Clean data before transformation
|
|
164
177
|
|
|
165
|
-
if self.preprocess:
|
|
166
|
-
# If the preprocess flag is true
|
|
178
|
+
if self.preprocess and self.operations:
|
|
167
179
|
print_message("Preprocessing data", self.logger, SCRIPT_NAME)
|
|
168
|
-
for operation in self.operations:
|
|
169
|
-
features = self.transform_seq[operation]["feature_list"]
|
|
170
180
|
|
|
171
|
-
|
|
172
|
-
|
|
181
|
+
for op_name, transformer in self.operations.items():
|
|
182
|
+
features = self.transform_seq[op_name]["feature_list"]
|
|
183
|
+
|
|
184
|
+
if isinstance(transformer, str):
|
|
185
|
+
transformed_data = bin_score(data[features].to_numpy())
|
|
173
186
|
else:
|
|
174
|
-
transformed_data =
|
|
175
|
-
|
|
176
|
-
)
|
|
187
|
+
transformed_data = transformer.transform(data[features])
|
|
188
|
+
|
|
177
189
|
data[features] = transformed_data
|
|
178
190
|
|
|
179
|
-
return data
|
|
191
|
+
return downcast_dtypes(data)
|