medpipe 0.2.2.dev0__tar.gz → 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of medpipe might be problematic. Click here for more details.

Files changed (57) hide show
  1. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/PKG-INFO +3 -3
  2. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/README.md +2 -1
  3. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/pyproject.toml +1 -1
  4. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/requirements.txt +0 -1
  5. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/__init__.py +2 -1
  6. medpipe-1.0.0/src/medpipe/_types.py +50 -0
  7. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/data/__init__.py +4 -3
  8. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/data/db.py +12 -10
  9. medpipe-1.0.0/src/medpipe/data/preprocessing.py +202 -0
  10. medpipe-0.2.2.dev0/src/medpipe/data/Preprocessor.py → medpipe-1.0.0/src/medpipe/data/preprocessor.py +65 -53
  11. medpipe-1.0.0/src/medpipe/data/sampler.py +741 -0
  12. medpipe-1.0.0/src/medpipe/data/utils.py +269 -0
  13. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/data/weighting.py +77 -67
  14. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/metrics/core.py +108 -140
  15. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/metrics/plots.py +54 -33
  16. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/models/__init__.py +1 -1
  17. medpipe-1.0.0/src/medpipe/models/calibrators.py +345 -0
  18. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/models/core.py +76 -54
  19. medpipe-1.0.0/src/medpipe/models/predictors.py +280 -0
  20. medpipe-1.0.0/src/medpipe/pipeline/__init__.py +8 -0
  21. medpipe-1.0.0/src/medpipe/pipeline/pipeline.py +833 -0
  22. medpipe-1.0.0/src/medpipe/py.typed +0 -0
  23. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/utils/config.py +18 -11
  24. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/utils/exceptions.py +23 -12
  25. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/utils/io.py +11 -5
  26. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/utils/logger.py +13 -6
  27. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe.egg-info/PKG-INFO +3 -3
  28. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe.egg-info/SOURCES.txt +12 -7
  29. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe.egg-info/requires.txt +0 -1
  30. medpipe-1.0.0/test/test_data_preprocessing.py +132 -0
  31. medpipe-0.2.2.dev0/test/test_data_Preprocessor.py → medpipe-1.0.0/test/test_data_preprocessor.py +9 -4
  32. medpipe-1.0.0/test/test_data_sampler.py +667 -0
  33. medpipe-0.2.2.dev0/test/test_data_preprocessing.py → medpipe-1.0.0/test/test_data_utils.py +94 -112
  34. medpipe-1.0.0/test/test_models_calibrators.py +38 -0
  35. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/test/test_models_core.py +28 -15
  36. medpipe-1.0.0/test/test_models_predictors.py +34 -0
  37. medpipe-0.2.2.dev0/test/test_pipeline_Pipeline.py → medpipe-1.0.0/test/test_pipeline_pipeline.py +2 -50
  38. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/test/test_utils_exceptions.py +4 -2
  39. medpipe-0.2.2.dev0/src/medpipe/data/preprocessing.py +0 -320
  40. medpipe-0.2.2.dev0/src/medpipe/data/sampler.py +0 -561
  41. medpipe-0.2.2.dev0/src/medpipe/models/Calibrator.py +0 -147
  42. medpipe-0.2.2.dev0/src/medpipe/models/Predictor.py +0 -144
  43. medpipe-0.2.2.dev0/src/medpipe/pipeline/Pipeline.py +0 -832
  44. medpipe-0.2.2.dev0/src/medpipe/pipeline/__init__.py +0 -8
  45. medpipe-0.2.2.dev0/test/test_data_sampler.py +0 -477
  46. medpipe-0.2.2.dev0/test/test_metrics_core.py +0 -44
  47. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/LICENSE +0 -0
  48. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/setup.cfg +0 -0
  49. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/metrics/__init__.py +0 -0
  50. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe/utils/__init__.py +0 -0
  51. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe.egg-info/dependency_links.txt +0 -0
  52. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/src/medpipe.egg-info/top_level.txt +0 -0
  53. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/test/test_data_db.py +0 -0
  54. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/test/test_data_weighting.py +0 -0
  55. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/test/test_utils_config.py +0 -0
  56. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/test/test_utils_io.py +0 -0
  57. {medpipe-0.2.2.dev0 → medpipe-1.0.0}/test/test_utils_logger.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: medpipe
3
- Version: 0.2.2.dev0
3
+ Version: 1.0.0
4
4
  Summary: A package for clinical AI pipeline development
5
5
  Author-email: Mathias Roesler <mathias.roesler@auckland.co.nz>
6
6
  License-Expression: Apache-2.0
@@ -22,7 +22,6 @@ Requires-Dist: pytest==9.0.1
22
22
  Requires-Dist: scikit_learn==1.8.0
23
23
  Requires-Dist: scipy==1.17.0
24
24
  Requires-Dist: pyarrow==23.0.1
25
- Requires-Dist: lightgbm==4.6.0
26
25
  Dynamic: license-file
27
26
 
28
27
  # medpipe
@@ -46,10 +45,11 @@ The **medpipe** package is a layer to help create AI models for clinical applica
46
45
  ___
47
46
  ## Installation
48
47
 
49
- To install **medpipe** from the PyPI use pip:
48
+ To install **medpipe** use pip:
50
49
  ```
51
50
  $ pip install medpipe
52
51
  ```
52
+
53
53
  or clone the GitHub repository and install the package with pip:
54
54
  ```
55
55
  $ git clone git@github.com:Surgical-Recovery-and-Safety-Lab/medpipe.git
@@ -19,10 +19,11 @@ The **medpipe** package is a layer to help create AI models for clinical applica
19
19
  ___
20
20
  ## Installation
21
21
 
22
- To install **medpipe** from the PyPI use pip:
22
+ To install **medpipe** use pip:
23
23
  ```
24
24
  $ pip install medpipe
25
25
  ```
26
+
26
27
  or clone the GitHub repository and install the package with pip:
27
28
  ```
28
29
  $ git clone git@github.com:Surgical-Recovery-and-Safety-Lab/medpipe.git
@@ -4,7 +4,7 @@ requires = [ "setuptools>=80.9" ]
4
4
 
5
5
  [project]
6
6
  name = "medpipe"
7
- version = "0.2.2.dev0"
7
+ version = "1.0.0"
8
8
  description = "A package for clinical AI pipeline development"
9
9
  readme = "README.md"
10
10
  license = "Apache-2.0"
@@ -7,4 +7,3 @@ pytest==9.0.1
7
7
  scikit_learn==1.8.0
8
8
  scipy==1.17.0
9
9
  pyarrow==23.0.1
10
- lightgbm==4.6.0
@@ -21,7 +21,7 @@ from .metrics import (
21
21
  print_metrics_CI,
22
22
  )
23
23
  from .models import get_full_proba, get_positive_proba, load_pipeline, save_pipeline
24
- from .pipeline.Pipeline import Pipeline
24
+ from .pipeline.pipeline import Pipeline
25
25
  from .utils import (
26
26
  exception_handler,
27
27
  load_data_from_csv,
@@ -52,3 +52,4 @@ __all__ = [
52
52
  "extract_data_from_db",
53
53
  "extract_labels",
54
54
  ]
55
+ __version__ = "v1.0.0"
@@ -0,0 +1,50 @@
1
+ """
2
+ Medpipe typings.
3
+
4
+ This module provides special types defined for the medpipe package.
5
+
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import TYPE_CHECKING, Annotated, Literal, Sequence, TypeAlias, TypeVar
11
+
12
+ import numpy as np
13
+ import numpy.typing as npt
14
+ import pandas as pd
15
+ from sklearn.ensemble import HistGradientBoostingClassifier
16
+ from sklearn.isotonic import IsotonicRegression
17
+ from sklearn.linear_model import LogisticRegression
18
+ from sklearn.preprocessing import OrdinalEncoder, PowerTransformer, StandardScaler
19
+
20
+ if TYPE_CHECKING:
21
+ from medpipe.models.calibrators import IsotonicCalibrator, LogisticCalibrator
22
+ from medpipe.models.predictors import HGBClassifier
23
+
24
+ # Define data and label types
25
+ SingleClassLabels: TypeAlias = Annotated[npt.NDArray[np.integer], Literal["N"]]
26
+ MultiClassLabels: TypeAlias = Annotated[npt.NDArray[np.integer], Literal["N", "C"]]
27
+ FullProba: TypeAlias = Annotated[npt.NDArray[np.number], Literal["C", "N", 2]]
28
+ PosProba: TypeAlias = Annotated[npt.NDArray[np.number], Literal["N", "C"]]
29
+ Labels: TypeAlias = SingleClassLabels | MultiClassLabels
30
+ PredData: TypeAlias = pd.DataFrame | npt.NDArray # Predictor class data
31
+ Data: TypeAlias = PredData | PosProba # Pipeline class data
32
+
33
+ # Define preprocessing types
34
+ PreprocessOpConfig: TypeAlias = dict[str, dict[str, list[str]]]
35
+ PreprocessOp: TypeAlias = OrdinalEncoder | StandardScaler | PowerTransformer
36
+
37
+ # Define metric types
38
+ CI: TypeAlias = tuple[npt.NDArray, npt.NDArray, npt.NDArray]
39
+ CIDict: TypeAlias = dict[str, CI]
40
+ MetricDict: TypeAlias = dict[str, list[float]]
41
+ ModelMetrics: TypeAlias = dict[str, dict[str, Sequence[float]]]
42
+
43
+ # Define model types
44
+ Classifier: TypeAlias = HistGradientBoostingClassifier
45
+ Predictor: TypeAlias = "HGBClassifier"
46
+ Regressor: TypeAlias = LogisticRegression | IsotonicRegression
47
+ Calibrator: TypeAlias = "IsotonicCalibrator | LogisticCalibrator"
48
+ Model: TypeAlias = Classifier | Regressor
49
+ R = TypeVar("R", bound=Regressor) # Generic Type Variable for Regressors
50
+ C = TypeVar("C", bound=Classifier) # Generic Type Variable for Classifiers
@@ -6,9 +6,10 @@ submodules:
6
6
  - preprocessing: contains functions for preprocessing data.
7
7
  - weighting: contains functions to compute sample weights for imbalanced data.
8
8
  - sampler: contains functions to sample the data for imbalanced data.
9
- - Preprocessor: class to prepare data for fitting.
9
+ - preprocessor: class to prepare data for fitting.
10
+ - utils: utility functions for data manipulation.
10
11
  """
11
12
 
12
- from . import Preprocessor, db, preprocessing, sampler, weighting
13
+ from . import db, preprocessing, preprocessor, sampler, utils, weighting
13
14
  from .db import extract_data_from_db
14
- from .preprocessing import extract_labels
15
+ from .utils import extract_labels
@@ -8,11 +8,13 @@ Functions:
8
8
  - extract_data_from_db: Queries a SQL .db to extract data.
9
9
  """
10
10
 
11
+ from __future__ import annotations
12
+
11
13
  import sqlite3
12
14
 
13
15
  import pandas as pd
14
16
 
15
- import medpipe.utils.exceptions
17
+ from medpipe.utils.exceptions import file_checks
16
18
 
17
19
 
18
20
  def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") -> None:
@@ -21,11 +23,11 @@ def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") ->
21
23
 
22
24
  Parameters
23
25
  ----------
24
- parquet_file
26
+ parquet_file : str
25
27
  File path to the .parquet file.
26
- db_file
28
+ db_file : str
27
29
  File path to the .db file.
28
- table_name : default: 'main'
30
+ table_name : str, default: 'main'
29
31
  Name of the table to create in the SQL database.
30
32
 
31
33
  Returns
@@ -48,12 +50,12 @@ def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") ->
48
50
 
49
51
  """
50
52
  try:
51
- medpipe.utils.exceptions.file_checks(parquet_file, ".parquet")
53
+ file_checks(parquet_file, ".parquet")
52
54
  except (FileNotFoundError, IsADirectoryError, TypeError, ValueError):
53
55
  raise
54
56
 
55
57
  try:
56
- medpipe.utils.exceptions.file_checks(db_file, ".db", exists=False)
58
+ file_checks(db_file, ".db", exists=False)
57
59
  except (FileNotFoundError, TypeError, ValueError, IsADirectoryError):
58
60
  raise
59
61
 
@@ -69,7 +71,7 @@ def parquet_to_db(parquet_file: str, db_file: str, table_name: str = "main") ->
69
71
  conn.close()
70
72
 
71
73
 
72
- def extract_data_from_db(db_file: str, query: str):
74
+ def extract_data_from_db(db_file: str, query: str) -> pd.DataFrame:
73
75
  """
74
76
  Extracts data from a .db and saves it to a .csv file.
75
77
 
@@ -77,9 +79,9 @@ def extract_data_from_db(db_file: str, query: str):
77
79
 
78
80
  Parameters
79
81
  ----------
80
- db_file
82
+ db_file : str
81
83
  Path to the .db file.
82
- query
84
+ query : str
83
85
  Query to send to extract data.
84
86
 
85
87
  Returns
@@ -100,7 +102,7 @@ def extract_data_from_db(db_file: str, query: str):
100
102
 
101
103
  """
102
104
  try:
103
- medpipe.utils.exceptions.file_checks(db_file, ".db")
105
+ file_checks(db_file, ".db")
104
106
  except (FileNotFoundError, IsADirectoryError, TypeError, ValueError):
105
107
  raise
106
108
 
@@ -0,0 +1,202 @@
1
+ """
2
+ Preprocessing functions module.
3
+
4
+ This module provides functions to preprocess data before training.
5
+
6
+ Functions:
7
+ - train_test_it: Creates a KFold iterator to split data into test and train sets.
8
+ - convert_object_to_categorical: Converts object columns to categoricals.
9
+ - fit_preprocess_operations: Fits processing operations to data.
10
+ - bin_score: Bins the M3 score into 5 categories.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from typing import TYPE_CHECKING, Any, Mapping
16
+
17
+ import numpy as np
18
+ import pandas as pd
19
+ from sklearn.model_selection import GroupKFold, StratifiedKFold
20
+ from sklearn.preprocessing import OrdinalEncoder, PowerTransformer, StandardScaler
21
+
22
+ from medpipe._types import PreprocessOp, PreprocessOpConfig
23
+
24
+ if TYPE_CHECKING:
25
+ import numpy.typing as npt
26
+
27
+
28
+ def train_test_it(
29
+ group_k_fold: bool = False, **kwargs: Any
30
+ ) -> StratifiedKFold | GroupKFold:
31
+ """
32
+ Creates a KFold iterator to split data into test and train sets.
33
+
34
+ Parameters
35
+ ----------
36
+ group_k_fold : bool, default: False
37
+ If True, the data will be split using a group and a
38
+ GroupKFold iterator is returned.
39
+ **kwargs: Any
40
+ Extra arguments for the StratifiedKFold or GroupKFold class.
41
+
42
+ Returns
43
+ -------
44
+ kfold_it : StratifiedKFold | GroupKFold
45
+ KFold iterator.
46
+
47
+ Raises
48
+ ------
49
+ ValueError
50
+ If n_splits is less than 2.
51
+
52
+ """
53
+ # Create the correct argument dict for StratifiedKFold
54
+ args_dict = dict()
55
+ for key, value in kwargs.items():
56
+ match key:
57
+ case "random_state":
58
+ if value == -1:
59
+ value = None
60
+ args_dict.update({key: value})
61
+
62
+ case "shuffle":
63
+ args_dict.update({key: value})
64
+
65
+ case "n_splits":
66
+ if value < 2:
67
+ raise ValueError(
68
+ f"n_splits should be greater than 2, but got {value}"
69
+ )
70
+
71
+ args_dict.update({key: value})
72
+
73
+ if not group_k_fold:
74
+ kfold_it = StratifiedKFold(**args_dict)
75
+ else:
76
+ kfold_it = GroupKFold(**args_dict)
77
+
78
+ return kfold_it
79
+
80
+
81
+ def convert_object_to_categorical(data: pd.DataFrame) -> pd.DataFrame:
82
+ """
83
+ Converts all object columns of a DataFrame to categoricals.
84
+
85
+ Parameters
86
+ ----------
87
+ data : pd.DataFrame
88
+ DataFrame to manipulate.
89
+
90
+ Returns
91
+ -------
92
+ processed_data : pd.DataFrame
93
+ Processed DataFrame.
94
+
95
+ Raises
96
+ ------
97
+ TypeError
98
+ If data is not a pd.DataFrame.
99
+
100
+ """
101
+ if not isinstance(data, pd.DataFrame):
102
+ raise TypeError(f"data should be a pd.DataFrame, but got {type(data).__name__}")
103
+
104
+ processed_data = data.copy()
105
+ obj_cols = processed_data.select_dtypes(include=["object"]).columns
106
+
107
+ # Vectorized conversion for all object columns at once
108
+ if not obj_cols.empty:
109
+ processed_data[obj_cols] = processed_data[obj_cols].astype("category")
110
+
111
+ return processed_data
112
+
113
+
114
+ def fit_preprocess_operations(
115
+ data: pd.DataFrame, preprocessing_dict: PreprocessOpConfig
116
+ ) -> Mapping[str, PreprocessOp | str]:
117
+ """
118
+ Fits processing operations to data.
119
+
120
+ Parameters
121
+ ----------
122
+ data : pd.DataFrame
123
+ DataFrame to manipulate.
124
+ preprocessing_dict : PreprocessOpConfig
125
+ Dictionary of the operations and the features on which to operate.
126
+
127
+ Returns
128
+ -------
129
+ operation_dict : Mapping[str, PreprocessOp | str]
130
+ Dictionary of the different preprocessing objects.
131
+
132
+ Raises
133
+ ------
134
+ TypeError
135
+ If data is not a pd.DataFrame.
136
+ If features is not a list[str].
137
+ KeyError
138
+ If a features is not a valid key.
139
+ ValueError
140
+ If preprocess is not a valid preprocessing function.
141
+
142
+ """
143
+ if not isinstance(data, pd.DataFrame):
144
+ raise TypeError(f"data should be a pd.DataFrame, but got {type(data).__name__}")
145
+
146
+ # Use shallow copy instead of deepcopy for performance
147
+ data_working = data.copy()
148
+ operation_dict: dict[str, PreprocessOp | str] = {}
149
+
150
+ # Map names to classes for cleaner logic
151
+ transformers = {
152
+ "ordinal_encoder": OrdinalEncoder,
153
+ "standardise": StandardScaler,
154
+ "power_transform": PowerTransformer,
155
+ }
156
+
157
+ for op_name, config in preprocessing_dict.items():
158
+ features = config.get("feature_list")
159
+
160
+ if not isinstance(features, list):
161
+ raise TypeError(
162
+ f"features should be a list, but got {type(features).__name__}"
163
+ )
164
+
165
+ if features and not isinstance(features[0], str):
166
+ raise TypeError(
167
+ f"features must contain strings, got {type(features[0]).__name__}"
168
+ )
169
+
170
+ if op_name == "bin":
171
+ operation_dict[op_name] = "bin"
172
+ continue
173
+
174
+ if op_name in transformers:
175
+ # Instantiate, fit, and transform sequentially
176
+ model = transformers[op_name]()
177
+ operation_dict[op_name] = model.fit(data_working[features])
178
+
179
+ # Update working data so subsequent fits see transformed values
180
+ data_working[features] = model.transform(data_working[features])
181
+ else:
182
+ raise ValueError(f"{op_name} is an invalid preprocessing function")
183
+
184
+ return operation_dict
185
+
186
+
187
+ def bin_score(data: npt.NDArray) -> npt.NDArray:
188
+ """
189
+ Bins the M3 score into 5 categories.
190
+
191
+ Parameters
192
+ ----------
193
+ data : npt.NDArray
194
+ M3 score data.
195
+
196
+ Returns
197
+ -------
198
+ binned_data : npt.NDArray
199
+ Binned data.
200
+
201
+ """
202
+ return np.clip(np.ceil(data), 0, 4).astype(int)
@@ -5,8 +5,13 @@ This class creates a Preprocessor to prepare data.
5
5
 
6
6
  """
7
7
 
8
- from copy import deepcopy
8
+ from __future__ import annotations
9
9
 
10
+ from typing import TYPE_CHECKING, Mapping
11
+
12
+ import pandas as pd
13
+
14
+ from medpipe._types import PreprocessOp, PreprocessOpConfig
10
15
  from medpipe.utils.logger import print_message
11
16
 
12
17
  from .preprocessing import (
@@ -14,8 +19,14 @@ from .preprocessing import (
14
19
  convert_object_to_categorical,
15
20
  fit_preprocess_operations,
16
21
  )
22
+ from .utils import downcast_dtypes
23
+
24
+ if TYPE_CHECKING:
25
+ import logging
26
+
27
+ import pandas as pd
17
28
 
18
- SCRIPT_NAME = "data/Preprocessor"
29
+ SCRIPT_NAME = "data/preprocessor"
19
30
 
20
31
 
21
32
  class Preprocessor:
@@ -26,9 +37,11 @@ class Preprocessor:
26
37
  ----------
27
38
  preprocess : bool
28
39
  Flag to preprocess data or not.
29
- transform_seq : dict[str, dict[str,list[str]]]
40
+ transform_seq : PreprocessOpConfig
30
41
  Transformation sequence for the data.
31
- logger : logging.Logger or None, default: None
42
+ operations : Mapping[str, PreprocessorOp | str]
43
+ Preprocessing operations that are fitted.
44
+ logger : logging.Logger | None, default: None
32
45
  Logger object to log prints. If None print to terminal.
33
46
 
34
47
  Methods
@@ -45,15 +58,19 @@ class Preprocessor:
45
58
  Transforms input data based on fitted operations.
46
59
  """
47
60
 
48
- def __init__(self, preprocessor_config, logger=None):
61
+ def __init__(
62
+ self,
63
+ preprocessor_config: PreprocessOpConfig,
64
+ logger: logging.Logger | None = None,
65
+ ) -> None:
49
66
  """
50
67
  Initialise a Preprocessor class instance.
51
68
 
52
69
  Parameters
53
70
  ----------
54
- preprocessor_config : dict[str, dict[str, list[str]]]
71
+ preprocessor_config : PreprocessorOpConfig
55
72
  Configuration parameters for the preprocessor object.
56
- logger : logging.Logger or None, default: None
73
+ logger : logging.Logger | None, default: None
57
74
  Logger object to log prints. If None print to terminal.
58
75
 
59
76
  Returns
@@ -64,10 +81,12 @@ class Preprocessor:
64
81
  """
65
82
  self.preprocess = preprocessor_config.pop("preprocess")
66
83
  self.transform_seq = preprocessor_config
67
- self.operations = dict() # Empty dict to contain operations
84
+ self.operations: Mapping[str, PreprocessOp | str] = (
85
+ dict()
86
+ ) # Empty dict to contain operations
68
87
  self.logger = logger
69
88
 
70
- def _clean_data(self, X):
89
+ def _clean_data(self, X: pd.DataFrame) -> pd.DataFrame:
71
90
  """
72
91
  Cleans data before transformation.
73
92
 
@@ -76,21 +95,22 @@ class Preprocessor:
76
95
 
77
96
  Parameters
78
97
  ----------
79
- X : pd.Dataframe of shape (n_samples, n_features)
80
- Data to clean.
98
+ X : pd.Dataframe
99
+ Data to clean of shape (n_samples, n_features).
81
100
 
82
101
  Returns
83
102
  -------
84
- data : pd.Dataframe of shape (n_samples, n_features)
85
- Cleaned data.
103
+ data : pd.Dataframe
104
+ Cleaned data of shape (n_samples, n_features).
86
105
 
87
106
  """
88
107
  # Convert objects to categorical (not saved so needs to be here)
89
108
  data = convert_object_to_categorical(X)
90
109
 
91
- # Remove NaN values
92
110
  nb_nan_rows = data.isna().any(axis=1).sum()
93
- data = data.dropna()
111
+
112
+ if nb_nan_rows > 0:
113
+ data = data.dropna()
94
114
 
95
115
  print_message(
96
116
  f"Dropped {nb_nan_rows} rows with NaN values", self.logger, SCRIPT_NAME
@@ -98,39 +118,32 @@ class Preprocessor:
98
118
 
99
119
  return data
100
120
 
101
- def fit_transform(self, X):
121
+ def fit_transform(self, X: pd.DataFrame) -> pd.DataFrame:
102
122
  """
103
123
  Fits the operations and transforms the input data.
104
124
 
105
125
  Parameters
106
126
  ----------
107
- X : pd.Dataframe of shape (n_samples, n_features)
108
- Data to clean.
127
+ X : pd.Dataframe
128
+ Data of shape (n_samples, n_features) to clean.
109
129
 
110
130
  Returns
111
131
  -------
112
- data : pd.Dataframe of shape (n_samples, n_features)
113
- Transformed data.
132
+ data : pd.Dataframe
133
+ Transformed data of shape (n_samples, n_features).
114
134
 
115
135
  """
116
- data = self._clean_data(deepcopy(X)) # Clean data before transformation
117
-
118
- if self.preprocess:
119
- # If the preprocess flag is true
120
- print_message("Fitting processing operations", self.logger, SCRIPT_NAME)
121
- self.operations = fit_preprocess_operations(data, self.transform_seq)
122
- data = self.transform(data)
136
+ self.fit(X) # Fit operations
137
+ return self.transform(X) # Transform data
123
138
 
124
- return data
125
-
126
- def fit(self, X):
139
+ def fit(self, X: pd.DataFrame) -> None:
127
140
  """
128
141
  Fits the operations based on input data.
129
142
 
130
143
  Parameters
131
144
  ----------
132
- X : pd.Dataframe of shape (n_samples, n_features)
133
- Data to clean.
145
+ X : pd.Dataframe
146
+ Data to clean of shape (n_samples, n_features).
134
147
 
135
148
  Returns
136
149
  -------
@@ -138,42 +151,41 @@ class Preprocessor:
138
151
  Nothings is returned.
139
152
 
140
153
  """
141
- data = self._clean_data(X) # Clean data before transformation
154
+ if not self.preprocess: # Return None if preprocess flag is False
155
+ return
142
156
 
143
- if self.preprocess:
144
- # If the preprocess flag is true
145
- print_message("Fitting preprocessing operations", self.logger, SCRIPT_NAME)
146
- self.operations = fit_preprocess_operations(data, self.transform_seq)
157
+ data = self._clean_data(X) # Clean data before transformation
158
+ print_message("Fitting preprocessing operations", self.logger, SCRIPT_NAME)
159
+ self.operations = fit_preprocess_operations(data, self.transform_seq)
147
160
 
148
- def transform(self, X):
161
+ def transform(self, X: pd.DataFrame) -> pd.DataFrame:
149
162
  """
150
163
  Transforms input data based on fitted operations.
151
164
 
152
165
  Parameters
153
166
  ----------
154
- X : pd.Dataframe of shape (n_samples, n_features)
155
- Data to clean.
167
+ X : pd.Dataframe
168
+ Data to clean of shape (n_samples, n_features).
156
169
 
157
170
  Returns
158
171
  -------
159
- data : pd.Dataframe of shape (n_samples, n_features)
160
- Transformed data.
172
+ data : pd.Dataframe
173
+ Transformed data of shape (n_samples, n_features).
161
174
 
162
175
  """
163
- data = self._clean_data(deepcopy(X)) # Clean data before transformation
176
+ data = self._clean_data(X.copy()) # Clean data before transformation
164
177
 
165
- if self.preprocess:
166
- # If the preprocess flag is true
178
+ if self.preprocess and self.operations:
167
179
  print_message("Preprocessing data", self.logger, SCRIPT_NAME)
168
- for operation in self.operations:
169
- features = self.transform_seq[operation]["feature_list"]
170
180
 
171
- if operation == "bin":
172
- transformed_data = bin_score(data[features])
181
+ for op_name, transformer in self.operations.items():
182
+ features = self.transform_seq[op_name]["feature_list"]
183
+
184
+ if isinstance(transformer, str):
185
+ transformed_data = bin_score(data[features].to_numpy())
173
186
  else:
174
- transformed_data = self.operations[operation].transform(
175
- data[features]
176
- )
187
+ transformed_data = transformer.transform(data[features])
188
+
177
189
  data[features] = transformed_data
178
190
 
179
- return data
191
+ return downcast_dtypes(data)