autoencodix 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of autoencodix might be problematic. Click here for more details.
- autoencodix/__init__.py +27 -0
- autoencodix/base/.ipynb_checkpoints/_base_pipeline-checkpoint.py +1274 -0
- autoencodix/base/.ipynb_checkpoints/_base_trainer-checkpoint.py +247 -0
- autoencodix/base/__init__.py +21 -0
- autoencodix/base/_base_autoencoder.py +138 -0
- autoencodix/base/_base_dataset.py +103 -0
- autoencodix/base/_base_evaluator.py +34 -0
- autoencodix/base/_base_loss.py +311 -0
- autoencodix/base/_base_pipeline.py +1355 -0
- autoencodix/base/_base_predictor.py +6 -0
- autoencodix/base/_base_preprocessor.py +1033 -0
- autoencodix/base/_base_trainer.py +259 -0
- autoencodix/base/_base_visualizer.py +578 -0
- autoencodix/base/sampleconfig.yaml +132 -0
- autoencodix/configs/__init__.py +21 -0
- autoencodix/configs/default_config.py +652 -0
- autoencodix/configs/disentanglix_config.py +15 -0
- autoencodix/configs/maskix_config.py +39 -0
- autoencodix/configs/ontix_config.py +35 -0
- autoencodix/configs/stackix_config.py +33 -0
- autoencodix/configs/vanillix_config.py +18 -0
- autoencodix/configs/varix_config.py +15 -0
- autoencodix/configs/xmodalix_config.py +43 -0
- autoencodix/disentanglix.py +72 -0
- autoencodix/evaluate/.ipynb_checkpoints/evaluate-checkpoint.py +532 -0
- autoencodix/evaluate/__init__.py +4 -0
- autoencodix/evaluate/_general_evaluator.py +730 -0
- autoencodix/evaluate/_xmodalix_evaluator.py +376 -0
- autoencodix/imagix.py +76 -0
- autoencodix/losses/disentanglix_loss.py +215 -0
- autoencodix/losses/maskix_loss.py +94 -0
- autoencodix/losses/vanillix_loss.py +28 -0
- autoencodix/losses/varix_loss.py +102 -0
- autoencodix/losses/xmodal_loss.py +304 -0
- autoencodix/maskix.py +108 -0
- autoencodix/modeling/.ipynb_checkpoints/__init__-checkpoint.py +16 -0
- autoencodix/modeling/.ipynb_checkpoints/_captum_forward-checkpoint.py +16 -0
- autoencodix/modeling/__init__.py +20 -0
- autoencodix/modeling/_captum_forward.py +20 -0
- autoencodix/modeling/_classifier.py +45 -0
- autoencodix/modeling/_imagevae_architecture.py +296 -0
- autoencodix/modeling/_imgfast_architecture.py +288 -0
- autoencodix/modeling/_layer_factory.py +71 -0
- autoencodix/modeling/_maskix_architecture.py +190 -0
- autoencodix/modeling/_ontix_architecture.py +305 -0
- autoencodix/modeling/_vanillix_architecture.py +136 -0
- autoencodix/modeling/_varix_architecture.py +192 -0
- autoencodix/notebooks/__init__.py +0 -0
- autoencodix/ontix.py +136 -0
- autoencodix/py.typed +0 -0
- autoencodix/stackix.py +122 -0
- autoencodix/trainers/__init__.py +6 -0
- autoencodix/trainers/_general_trainer.py +525 -0
- autoencodix/trainers/_maskix_trainer.py +116 -0
- autoencodix/trainers/_ontix_trainer.py +77 -0
- autoencodix/trainers/_stackix_orchestrator.py +522 -0
- autoencodix/trainers/_stackix_trainer.py +209 -0
- autoencodix/trainers/_xmodal_trainer.py +1006 -0
- autoencodix/utils/.ipynb_checkpoints/_explainer-checkpoint.py +118 -0
- autoencodix/utils/.ipynb_checkpoints/_model_output-checkpoint.py +30 -0
- autoencodix/utils/.ipynb_checkpoints/_utils-checkpoint.py +626 -0
- autoencodix/utils/.ipynb_checkpoints/adata_converter-checkpoint.py +99 -0
- autoencodix/utils/.ipynb_checkpoints/feature_importance-checkpoint.py +823 -0
- autoencodix/utils/__init__.py +22 -0
- autoencodix/utils/_annealer.py +193 -0
- autoencodix/utils/_bulkreader.py +232 -0
- autoencodix/utils/_explainer.py +246 -0
- autoencodix/utils/_imgreader.py +404 -0
- autoencodix/utils/_internals.py +23 -0
- autoencodix/utils/_llm_explainer.py +318 -0
- autoencodix/utils/_losses.py +20 -0
- autoencodix/utils/_model_output.py +30 -0
- autoencodix/utils/_result.py +393 -0
- autoencodix/utils/_screader.py +76 -0
- autoencodix/utils/_traindynamics.py +187 -0
- autoencodix/utils/_utils.py +690 -0
- autoencodix/utils/adata_converter.py +111 -0
- autoencodix/utils/example_data.py +398 -0
- autoencodix/utils/feature_importance.py +829 -0
- autoencodix/utils/prompts.py +21 -0
- autoencodix/vanillix.py +141 -0
- autoencodix/varix.py +80 -0
- autoencodix/visualize/__init__.py +5 -0
- autoencodix/visualize/_general_visualizer.py +913 -0
- autoencodix/visualize/_imagix_visualizer.py +111 -0
- autoencodix/visualize/_xmodal_visualizer.py +841 -0
- autoencodix/visualize/visualize.py +1214 -0
- autoencodix/xmodalix.py +318 -0
- autoencodix-0.2.0.dist-info/METADATA +143 -0
- autoencodix-0.2.0.dist-info/RECORD +91 -0
- autoencodix-0.2.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,532 @@
|
|
|
1
|
+
from typing import Any, Union
|
|
2
|
+
import warnings
|
|
3
|
+
|
|
4
|
+
import pandas as pd
|
|
5
|
+
from sklearn import linear_model
|
|
6
|
+
from sklearn.model_selection import cross_validate
|
|
7
|
+
from sklearn.decomposition import PCA
|
|
8
|
+
from sklearn.metrics import get_scorer
|
|
9
|
+
|
|
10
|
+
from umap import UMAP
|
|
11
|
+
from sklearn.manifold import TSNE
|
|
12
|
+
from sklearn.base import ClassifierMixin, RegressorMixin
|
|
13
|
+
|
|
14
|
+
from autoencodix.utils._result import Result
|
|
15
|
+
from autoencodix.data._datasetcontainer import DatasetContainer
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Evaluator:
|
|
19
|
+
def __init__(self):
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
def evaluate(
|
|
23
|
+
self,
|
|
24
|
+
datasets: DatasetContainer,
|
|
25
|
+
result: Result,
|
|
26
|
+
ml_model_class: ClassifierMixin = linear_model.LogisticRegression(), # Default is sklearn LogisticRegression
|
|
27
|
+
ml_model_regression: RegressorMixin = linear_model.LinearRegression(), # Default is sklearn LinearRegression
|
|
28
|
+
params: Union[
|
|
29
|
+
list, str
|
|
30
|
+
] = "all", # No default? ... or all params in annotation?
|
|
31
|
+
metric_class: str = "roc_auc_ovr", # Default is 'roc_auc_ovr' via https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-string-names
|
|
32
|
+
metric_regression: str = "r2", # Default is 'r2'
|
|
33
|
+
reference_methods: list = [], # Default [], Options are "PCA", "UMAP", "TSNE", "RandomFeature"
|
|
34
|
+
split_type: str = "use-split", # Default is "use-split", other options: "CV-5", ... "LOOCV"?
|
|
35
|
+
) -> Result:
|
|
36
|
+
"""
|
|
37
|
+
Evaluate the performance of machine learning models on various feature representations and clinical parameters.
|
|
38
|
+
This method performs classification or regression tasks using specified machine learning models on different feature sets (e.g., latent space, PCA, UMAP, TSNE, RandomFeature) and clinical annotation parameters. It supports multiple evaluation strategies, including pre-defined train/valid/test splits, k-fold cross-validation, and leave-one-out cross-validation. The results are aggregated and stored in the provided `result` object.
|
|
39
|
+
Parameters
|
|
40
|
+
----------
|
|
41
|
+
datasets : DatasetContainer
|
|
42
|
+
An DatasetContainer containing train, valid, and test datasets, each with `sample_ids` and `metadata` (either a DataFrame or a dictionary with a 'paired' key for clinical annotations).
|
|
43
|
+
result : Result
|
|
44
|
+
An Result object to store the evaluation results. Should have an `embedding_evaluation` attribute which updated (typically a DataFrame).
|
|
45
|
+
ml_model_class : ClassifierMixin, optional
|
|
46
|
+
The scikit-learn classifier to use for classification tasks (default: `sklearn.linear_model.LogisticRegression()`).
|
|
47
|
+
ml_model_regression : RegressorMixin, optional
|
|
48
|
+
The scikit-learn regressor to use for regression tasks (default: `sklearn.linear_model.LinearRegression()`).
|
|
49
|
+
params : list or str, optional
|
|
50
|
+
List of clinical annotation columns to evaluate, or "all" to use all columns (default: "all").
|
|
51
|
+
metric_class : str, optional
|
|
52
|
+
Scoring metric for classification tasks (default: "roc_auc_ovr").
|
|
53
|
+
metric_regression : str, optional
|
|
54
|
+
Scoring metric for regression tasks (default: "r2").
|
|
55
|
+
reference_methods : list, optional
|
|
56
|
+
List of feature representations to evaluate (e.g., "PCA", "UMAP", "TSNE", "RandomFeature"). "Latent" is always included (default: []).
|
|
57
|
+
split_type : str, optional
|
|
58
|
+
Evaluation strategy: "use-split" for pre-defined splits, "CV-N" for N-fold cross-validation, or "LOOCV" for leave-one-out cross-validation (default: "use-split").
|
|
59
|
+
Returns
|
|
60
|
+
-------
|
|
61
|
+
Result
|
|
62
|
+
The updated result object with evaluation results stored in `embedding_evaluation`.
|
|
63
|
+
Raises
|
|
64
|
+
------
|
|
65
|
+
ValueError
|
|
66
|
+
If required annotation data is missing or improperly formatted, or if an unsupported split type is specified.
|
|
67
|
+
Notes
|
|
68
|
+
-----
|
|
69
|
+
- Samples with missing annotation values for a given parameter are excluded from the corresponding evaluation.
|
|
70
|
+
- For "RandomFeature", five random feature sets are evaluated.
|
|
71
|
+
- The method appends results to any existing `embedding_evaluation` in the result object.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
already_warned = False
|
|
75
|
+
|
|
76
|
+
df_results = pd.DataFrame()
|
|
77
|
+
|
|
78
|
+
## TODO when x-modalix is ready, check how to adjust evaluation of both latent spaces
|
|
79
|
+
# if cfg["MODEL_TYPE"] == "x-modalix":
|
|
80
|
+
# if "Latent" in cfg["ML_TASKS"]:
|
|
81
|
+
# cfg["ML_TASKS"].remove("Latent")
|
|
82
|
+
# cfg["ML_TASKS"].append("Latent_FROM")
|
|
83
|
+
# cfg["ML_TASKS"].append("Latent_TO")
|
|
84
|
+
# cfg["ML_TASKS"].append("Latent_BOTH")
|
|
85
|
+
|
|
86
|
+
reference_methods.append("Latent")
|
|
87
|
+
|
|
88
|
+
for task in reference_methods:
|
|
89
|
+
print(f"Perform ML task with feature df: {task}")
|
|
90
|
+
|
|
91
|
+
## Getting clin_data
|
|
92
|
+
if hasattr(datasets.train, "metadata"):
|
|
93
|
+
# Check if metadata is a dictionary and contains 'paired'
|
|
94
|
+
if isinstance(datasets.train.metadata, dict):
|
|
95
|
+
if "paired" in datasets.train.metadata:
|
|
96
|
+
clin_data = datasets.train.metadata["paired"]
|
|
97
|
+
if hasattr(datasets, "test"):
|
|
98
|
+
clin_data = pd.concat(
|
|
99
|
+
[clin_data, datasets.test.metadata["paired"]],
|
|
100
|
+
axis=0,
|
|
101
|
+
)
|
|
102
|
+
if hasattr(datasets, "valid"):
|
|
103
|
+
clin_data = pd.concat(
|
|
104
|
+
[clin_data, datasets.valid.metadata["paired"]],
|
|
105
|
+
axis=0,
|
|
106
|
+
)
|
|
107
|
+
else:
|
|
108
|
+
# Raise error no annotation given
|
|
109
|
+
raise ValueError(
|
|
110
|
+
"Please provide paired annotation data with key 'paired' in metadata dictionary."
|
|
111
|
+
)
|
|
112
|
+
elif isinstance(datasets.train.metadata, pd.DataFrame):
|
|
113
|
+
clin_data = datasets.train.metadata
|
|
114
|
+
if hasattr(datasets, "test"):
|
|
115
|
+
clin_data = pd.concat(
|
|
116
|
+
[clin_data, datasets.test.metadata],
|
|
117
|
+
axis=0,
|
|
118
|
+
)
|
|
119
|
+
if hasattr(datasets, "valid"):
|
|
120
|
+
clin_data = pd.concat(
|
|
121
|
+
[clin_data, datasets.valid.metadata],
|
|
122
|
+
axis=0,
|
|
123
|
+
)
|
|
124
|
+
else:
|
|
125
|
+
# Raise error no annotation given
|
|
126
|
+
raise ValueError(
|
|
127
|
+
"Metadata is not a dictionary or DataFrame. Please provide a valid annotation data type."
|
|
128
|
+
)
|
|
129
|
+
else:
|
|
130
|
+
# Raise error no annotation given
|
|
131
|
+
raise ValueError(
|
|
132
|
+
"No annotation data found. Please provide a valid annotation data type."
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
if split_type == "use-split":
|
|
136
|
+
# Pandas dataframe with sample_ids and split information
|
|
137
|
+
sample_split = pd.DataFrame(columns=["SAMPLE_ID", "SPLIT"])
|
|
138
|
+
|
|
139
|
+
if hasattr(datasets, "train"):
|
|
140
|
+
sample_split_temp = dict(
|
|
141
|
+
sample_split,
|
|
142
|
+
**{
|
|
143
|
+
"SAMPLE_ID": datasets.train.sample_ids,
|
|
144
|
+
"SPLIT": ["train"] * len(datasets.train.sample_ids),
|
|
145
|
+
},
|
|
146
|
+
)
|
|
147
|
+
sample_split = pd.concat(
|
|
148
|
+
[sample_split, pd.DataFrame(sample_split_temp)],
|
|
149
|
+
axis=0,
|
|
150
|
+
ignore_index=True,
|
|
151
|
+
)
|
|
152
|
+
else:
|
|
153
|
+
raise ValueError(
|
|
154
|
+
"No training data found. Please provide a valid training dataset."
|
|
155
|
+
)
|
|
156
|
+
if hasattr(datasets, "valid"):
|
|
157
|
+
sample_split_temp = dict(
|
|
158
|
+
sample_split,
|
|
159
|
+
**{
|
|
160
|
+
"SAMPLE_ID": datasets.valid.sample_ids,
|
|
161
|
+
"SPLIT": ["valid"] * len(datasets.valid.sample_ids),
|
|
162
|
+
},
|
|
163
|
+
)
|
|
164
|
+
sample_split = pd.concat(
|
|
165
|
+
[sample_split, pd.DataFrame(sample_split_temp)],
|
|
166
|
+
axis=0,
|
|
167
|
+
ignore_index=True,
|
|
168
|
+
)
|
|
169
|
+
if hasattr(datasets, "test"):
|
|
170
|
+
sample_split_temp = dict(
|
|
171
|
+
sample_split,
|
|
172
|
+
**{
|
|
173
|
+
"SAMPLE_ID": datasets.test.sample_ids,
|
|
174
|
+
"SPLIT": ["test"] * len(datasets.test.sample_ids),
|
|
175
|
+
},
|
|
176
|
+
)
|
|
177
|
+
sample_split = pd.concat(
|
|
178
|
+
[sample_split, pd.DataFrame(sample_split_temp)],
|
|
179
|
+
axis=0,
|
|
180
|
+
ignore_index=True,
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
sample_split = sample_split.set_index("SAMPLE_ID", drop=False)
|
|
184
|
+
|
|
185
|
+
## df -> task
|
|
186
|
+
subtask = [task]
|
|
187
|
+
if task == "RandomFeature":
|
|
188
|
+
subtask = [task + str(x) for x in range(1, 6)]
|
|
189
|
+
for sub in subtask:
|
|
190
|
+
df = self._load_input_for_ml(task, datasets, result)
|
|
191
|
+
|
|
192
|
+
if params == "all":
|
|
193
|
+
params = clin_data.columns.tolist()
|
|
194
|
+
|
|
195
|
+
for task_param in params:
|
|
196
|
+
if "Latent" in task:
|
|
197
|
+
print(f"Perform ML task for target parameter: {task_param}")
|
|
198
|
+
## Check if classification or regression task
|
|
199
|
+
ml_type = self._get_ml_type(clin_data, task_param)
|
|
200
|
+
|
|
201
|
+
if pd.isna(clin_data[task_param]).sum() > 0:
|
|
202
|
+
|
|
203
|
+
if not already_warned:
|
|
204
|
+
print(
|
|
205
|
+
"There are NA values in the annotation file. Samples with missing data will be removed for ML task evaluation."
|
|
206
|
+
)
|
|
207
|
+
already_warned = True
|
|
208
|
+
# logger.warning(clin_data.loc[pd.isna(clin_data[task_param]), task_param])
|
|
209
|
+
|
|
210
|
+
samples_nonna = clin_data.loc[
|
|
211
|
+
pd.notna(clin_data[task_param]), task_param
|
|
212
|
+
].index
|
|
213
|
+
# print(df)
|
|
214
|
+
df = df.loc[samples_nonna.intersection(df.index), :]
|
|
215
|
+
if split_type == "use-split":
|
|
216
|
+
sample_split = sample_split.loc[
|
|
217
|
+
samples_nonna.intersection(sample_split.index), :
|
|
218
|
+
]
|
|
219
|
+
# print(sample_split)
|
|
220
|
+
|
|
221
|
+
if ml_type == "classification":
|
|
222
|
+
metric = metric_class
|
|
223
|
+
sklearn_ml = ml_model_class
|
|
224
|
+
|
|
225
|
+
if ml_type == "regression":
|
|
226
|
+
metric = metric_regression
|
|
227
|
+
sklearn_ml = ml_model_regression
|
|
228
|
+
|
|
229
|
+
if split_type == "use-split":
|
|
230
|
+
results = self._single_ml_presplit(
|
|
231
|
+
sample_split=sample_split,
|
|
232
|
+
df=df,
|
|
233
|
+
clin_data=clin_data,
|
|
234
|
+
task_param=task_param,
|
|
235
|
+
sklearn_ml=sklearn_ml,
|
|
236
|
+
metric=metric,
|
|
237
|
+
ml_type=ml_type,
|
|
238
|
+
)
|
|
239
|
+
elif split_type.startswith("CV-"):
|
|
240
|
+
cv_folds = int(split_type.split("-")[1])
|
|
241
|
+
|
|
242
|
+
results = self._single_ml(
|
|
243
|
+
df=df,
|
|
244
|
+
clin_data=clin_data,
|
|
245
|
+
task_param=task_param,
|
|
246
|
+
sklearn_ml=sklearn_ml,
|
|
247
|
+
metric=metric,
|
|
248
|
+
cv_folds=cv_folds,
|
|
249
|
+
)
|
|
250
|
+
elif split_type == "LOOCV":
|
|
251
|
+
# Leave One Out Cross Validation
|
|
252
|
+
results = self._single_ml(
|
|
253
|
+
df=df,
|
|
254
|
+
clin_data=clin_data,
|
|
255
|
+
task_param=task_param,
|
|
256
|
+
sklearn_ml=sklearn_ml,
|
|
257
|
+
metric=metric,
|
|
258
|
+
cv_folds=len(df),
|
|
259
|
+
)
|
|
260
|
+
else:
|
|
261
|
+
raise ValueError(
|
|
262
|
+
f"Your split type {split_type} is not supported. Please use 'use-split', 'CV-5', 'LOOCV' or 'CV-N'."
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
res_ml_alg = [str(sklearn_ml) for x in range(0, results.shape[0])]
|
|
266
|
+
res_ml_type = [ml_type for x in range(0, results.shape[0])]
|
|
267
|
+
res_ml_task = [task for x in range(0, results.shape[0])]
|
|
268
|
+
res_ml_subtask = [sub for x in range(0, results.shape[0])]
|
|
269
|
+
|
|
270
|
+
results["ML_ALG"] = res_ml_alg
|
|
271
|
+
results["ML_TYPE"] = res_ml_type
|
|
272
|
+
results["ML_TASK"] = res_ml_task
|
|
273
|
+
results["ML_SUBTASK"] = res_ml_subtask
|
|
274
|
+
|
|
275
|
+
df_results = pd.concat([df_results, results])
|
|
276
|
+
|
|
277
|
+
## Check if embedding_evaluation is empty
|
|
278
|
+
if (
|
|
279
|
+
hasattr(result, "embedding_evaluation")
|
|
280
|
+
and len(result.embedding_evaluation) == 0
|
|
281
|
+
):
|
|
282
|
+
result.embedding_evaluation = df_results
|
|
283
|
+
else:
|
|
284
|
+
# merge with existing results
|
|
285
|
+
result.embedding_evaluation = pd.concat(
|
|
286
|
+
[result.embedding_evaluation, df_results], axis=0
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
return result
|
|
290
|
+
|
|
291
|
+
@staticmethod
|
|
292
|
+
def _single_ml(
|
|
293
|
+
df: pd.DataFrame,
|
|
294
|
+
clin_data: pd.DataFrame,
|
|
295
|
+
task_param: str,
|
|
296
|
+
sklearn_ml: Union[ClassifierMixin, RegressorMixin],
|
|
297
|
+
metric: str,
|
|
298
|
+
cv_folds: int = 5,
|
|
299
|
+
):
|
|
300
|
+
"""
|
|
301
|
+
Function learns on the given data frame df and label data the provided sklearn model.
|
|
302
|
+
Cross validation is performed according to the config and scores are returned as output as specified by metrics
|
|
303
|
+
ARGS:
|
|
304
|
+
df (pd.DataFrame): Dataframe with input data
|
|
305
|
+
clin_data (pd.DataFrame): Dataframe with label data
|
|
306
|
+
task_param (str): Column name with label data
|
|
307
|
+
sklearn_ml (sklearn.module): Sklearn ML module specifying the ML algorithm
|
|
308
|
+
metric (str): string specifying the metric to be calculated by cross validation
|
|
309
|
+
RETURNS:
|
|
310
|
+
score_df (pd.DataFrame): data frame containing metrics (scores) for all CV runs (long format)
|
|
311
|
+
|
|
312
|
+
"""
|
|
313
|
+
|
|
314
|
+
# X -> df
|
|
315
|
+
# Y -> task_param
|
|
316
|
+
y = clin_data.loc[df.index, task_param]
|
|
317
|
+
score_df = dict()
|
|
318
|
+
|
|
319
|
+
## Cross Validation
|
|
320
|
+
if len(y.unique()) > 1:
|
|
321
|
+
scores = cross_validate(
|
|
322
|
+
sklearn_ml, df, y, cv=cv_folds, scoring=metric, return_train_score=True
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
# Output
|
|
326
|
+
|
|
327
|
+
# Output Format
|
|
328
|
+
# CV_RUN | SCORE_SPLIT | TASK_PARAM | METRIC | VALUE
|
|
329
|
+
|
|
330
|
+
score_df["cv_run"] = list()
|
|
331
|
+
score_df["score_split"] = list()
|
|
332
|
+
score_df["CLINIC_PARAM"] = list()
|
|
333
|
+
score_df["metric"] = list()
|
|
334
|
+
score_df["value"] = list()
|
|
335
|
+
|
|
336
|
+
cv_runs = ["CV_" + str(x) for x in range(1, cv_folds + 1)]
|
|
337
|
+
task_param_cv = [task_param for x in range(1, cv_folds + 1)]
|
|
338
|
+
|
|
339
|
+
for m in scores:
|
|
340
|
+
if m.split("_")[0] == "test" or m.split("_")[0] == "train":
|
|
341
|
+
split_cv = [m.split("_")[0] for x in range(1, cv_folds + 1)]
|
|
342
|
+
metric_cv = [metric for x in range(1, cv_folds + 1)]
|
|
343
|
+
|
|
344
|
+
score_df["cv_run"].extend(cv_runs)
|
|
345
|
+
score_df["score_split"].extend(split_cv)
|
|
346
|
+
score_df["CLINIC_PARAM"].extend(task_param_cv)
|
|
347
|
+
score_df["metric"].extend(metric_cv)
|
|
348
|
+
score_df["value"].extend(scores[m])
|
|
349
|
+
|
|
350
|
+
return pd.DataFrame(score_df)
|
|
351
|
+
|
|
352
|
+
@staticmethod
|
|
353
|
+
def _single_ml_presplit(
|
|
354
|
+
sample_split: pd.DataFrame,
|
|
355
|
+
df: pd.DataFrame,
|
|
356
|
+
clin_data: pd.DataFrame,
|
|
357
|
+
task_param: str,
|
|
358
|
+
sklearn_ml: Union[ClassifierMixin, RegressorMixin],
|
|
359
|
+
metric: str,
|
|
360
|
+
ml_type: str,
|
|
361
|
+
):
|
|
362
|
+
"""
|
|
363
|
+
Trains the provided sklearn model on the training split and evaluates it on train, valid, and test splits using the specified metric.
|
|
364
|
+
|
|
365
|
+
Parameters
|
|
366
|
+
----------
|
|
367
|
+
sample_split : pd.DataFrame
|
|
368
|
+
DataFrame with sample IDs and their corresponding split ("train", "valid", "test").
|
|
369
|
+
df : pd.DataFrame
|
|
370
|
+
DataFrame with input features, indexed by sample IDs.
|
|
371
|
+
clin_data : pd.DataFrame
|
|
372
|
+
DataFrame with label/annotation data, indexed by sample IDs.
|
|
373
|
+
task_param : str
|
|
374
|
+
Column name in clin_data specifying the target variable.
|
|
375
|
+
sklearn_ml : ClassifierMixin or RegressorMixin
|
|
376
|
+
Instantiated sklearn model to use for training and evaluation.
|
|
377
|
+
metric : str
|
|
378
|
+
Scoring metric compatible with sklearn's get_scorer.
|
|
379
|
+
ml_type : str
|
|
380
|
+
Type of machine learning task ("classification" or "regression").
|
|
381
|
+
|
|
382
|
+
Returns
|
|
383
|
+
-------
|
|
384
|
+
pd.DataFrame
|
|
385
|
+
DataFrame containing evaluation scores for each split (train, valid, test) and the specified metric.
|
|
386
|
+
|
|
387
|
+
Raises
|
|
388
|
+
------
|
|
389
|
+
ValueError
|
|
390
|
+
If the provided metric is not supported by sklearn.
|
|
391
|
+
"""
|
|
392
|
+
split_list = ["train", "valid", "test"]
|
|
393
|
+
|
|
394
|
+
score_df = dict()
|
|
395
|
+
score_df["score_split"] = list()
|
|
396
|
+
score_df["CLINIC_PARAM"] = list()
|
|
397
|
+
score_df["metric"] = list()
|
|
398
|
+
score_df["value"] = list()
|
|
399
|
+
|
|
400
|
+
X_train = df.loc[
|
|
401
|
+
sample_split.loc[sample_split.SPLIT == "train", "SAMPLE_ID"], :
|
|
402
|
+
]
|
|
403
|
+
train_samples = [s for s in X_train.index]
|
|
404
|
+
Y_train = clin_data.loc[train_samples, task_param]
|
|
405
|
+
# train model once on training data
|
|
406
|
+
if len(Y_train.unique()) > 1:
|
|
407
|
+
sklearn_ml.fit(X_train, Y_train)
|
|
408
|
+
|
|
409
|
+
# eval on all splits
|
|
410
|
+
for split in split_list:
|
|
411
|
+
X = df.loc[
|
|
412
|
+
sample_split.loc[sample_split.SPLIT == split, "SAMPLE_ID"], :
|
|
413
|
+
]
|
|
414
|
+
samples = [s for s in X.index]
|
|
415
|
+
Y = clin_data.loc[samples, task_param]
|
|
416
|
+
|
|
417
|
+
# Performace on train, valid and test data split
|
|
418
|
+
|
|
419
|
+
score_df["score_split"].append(split)
|
|
420
|
+
score_df["CLINIC_PARAM"].append(task_param)
|
|
421
|
+
score_df["metric"].append(metric)
|
|
422
|
+
sklearn_scorer = get_scorer(metric)
|
|
423
|
+
|
|
424
|
+
if sklearn_scorer is None:
|
|
425
|
+
raise ValueError(
|
|
426
|
+
f"Your metric {metric} is not supported by sklearn. Please use a valid metric."
|
|
427
|
+
)
|
|
428
|
+
|
|
429
|
+
if ml_type == "classification":
|
|
430
|
+
# Check that Y has only classes which are present in Y_train
|
|
431
|
+
if len(set(Y.unique()).difference(set(Y_train.unique()))) > 0:
|
|
432
|
+
print(
|
|
433
|
+
f"Classes in split {split} are not present in training data"
|
|
434
|
+
)
|
|
435
|
+
# Adjust Y to only contain classes present in Y_train
|
|
436
|
+
Y = Y[Y.isin(Y_train.unique())]
|
|
437
|
+
# Adjust X as well
|
|
438
|
+
X = X.loc[Y.index, :]
|
|
439
|
+
|
|
440
|
+
score_temp = sklearn_scorer(sklearn_ml, X, Y)
|
|
441
|
+
score_df["value"].append(score_temp)
|
|
442
|
+
else:
|
|
443
|
+
## Warning that there is only one class in the training data
|
|
444
|
+
warnings.warn(
|
|
445
|
+
f"Warning: There is only one class in the training data for task parameter {task_param}. Skipping evaluation for this task."
|
|
446
|
+
)
|
|
447
|
+
|
|
448
|
+
return pd.DataFrame(score_df)
|
|
449
|
+
|
|
450
|
+
@staticmethod
|
|
451
|
+
def _get_ml_type(clin_data: pd.DataFrame, task_param: str) -> str:
|
|
452
|
+
"""
|
|
453
|
+
Determines the machine learning task type (classification or regression) based on the data type of a specified column in clinical data.
|
|
454
|
+
|
|
455
|
+
Args:
|
|
456
|
+
clin_data (pd.DataFrame): The clinical data as a pandas DataFrame.
|
|
457
|
+
task_param (str): The column name in clin_data to inspect for determining the task type.
|
|
458
|
+
|
|
459
|
+
Returns:
|
|
460
|
+
str: "classification" if the first value in the specified column is a string, otherwise "regression".
|
|
461
|
+
"""
|
|
462
|
+
## Auto-Detection
|
|
463
|
+
if type(list(clin_data[task_param])[0]) is str:
|
|
464
|
+
ml_type = "classification"
|
|
465
|
+
else:
|
|
466
|
+
ml_type = "regression"
|
|
467
|
+
|
|
468
|
+
return ml_type
|
|
469
|
+
|
|
470
|
+
@staticmethod
|
|
471
|
+
def _load_input_for_ml(
|
|
472
|
+
task: str, dataset: DatasetContainer, result: Result
|
|
473
|
+
) -> pd.DataFrame:
|
|
474
|
+
"""
|
|
475
|
+
Loads and processes input data for various machine learning tasks based on the specified task type.
|
|
476
|
+
Parameters:
|
|
477
|
+
task (str): The type of ML task. Supported values are "Latent", "UMAP", "PCA", "TSNE", and "RandomFeature".
|
|
478
|
+
dataset (DatasetContainer): The dataset container object holding train, validation, and test splits.
|
|
479
|
+
result (Result): The result object containing model configuration and methods to retrieve latent representations.
|
|
480
|
+
Returns:
|
|
481
|
+
pd.DataFrame: A DataFrame containing the processed input data suitable for the specified ML task.
|
|
482
|
+
Raises:
|
|
483
|
+
ValueError: If the provided task is not supported.
|
|
484
|
+
Task Details:
|
|
485
|
+
- "Latent": Concatenates latent representations from train, validation, and test splits at the final epoch.
|
|
486
|
+
- "UMAP": Applies UMAP dimensionality reduction to the concatenated dataset splits.
|
|
487
|
+
- "PCA": Applies PCA dimensionality reduction to the concatenated dataset splits.
|
|
488
|
+
- "TSNE": Applies t-SNE dimensionality reduction to the concatenated dataset splits.
|
|
489
|
+
- "RandomFeature": Randomly samples columns (features) from the concatenated dataset splits.
|
|
490
|
+
"""
|
|
491
|
+
|
|
492
|
+
final_epoch = result.model.config.epochs - 1
|
|
493
|
+
|
|
494
|
+
if task == "Latent":
|
|
495
|
+
df = pd.concat(
|
|
496
|
+
[
|
|
497
|
+
result.get_latent_df(epoch=final_epoch, split="train"),
|
|
498
|
+
result.get_latent_df(epoch=final_epoch, split="valid"),
|
|
499
|
+
result.get_latent_df(epoch=-1, split="test"),
|
|
500
|
+
]
|
|
501
|
+
)
|
|
502
|
+
elif task in ["UMAP", "PCA", "TSNE", "RandomFeature"]:
|
|
503
|
+
df_processed = pd.concat(
|
|
504
|
+
[
|
|
505
|
+
dataset.train._to_df(),
|
|
506
|
+
dataset.test._to_df(),
|
|
507
|
+
dataset.valid._to_df(),
|
|
508
|
+
]
|
|
509
|
+
)
|
|
510
|
+
if task == "UMAP":
|
|
511
|
+
reducer = UMAP(n_components=result.model.config.latent_dim)
|
|
512
|
+
df = pd.DataFrame(
|
|
513
|
+
reducer.fit_transform(df_processed), index=df_processed.index
|
|
514
|
+
)
|
|
515
|
+
elif task == "PCA":
|
|
516
|
+
reducer = PCA(n_components=result.model.config.latent_dim)
|
|
517
|
+
df = pd.DataFrame(
|
|
518
|
+
reducer.fit_transform(df_processed), index=df_processed.index
|
|
519
|
+
)
|
|
520
|
+
elif task == "TSNE":
|
|
521
|
+
reducer = TSNE(n_components=result.model.config.latent_dim)
|
|
522
|
+
df = pd.DataFrame(
|
|
523
|
+
reducer.fit_transform(df_processed), index=df_processed.index
|
|
524
|
+
)
|
|
525
|
+
elif task == "RandomFeature":
|
|
526
|
+
df = df_processed.sample(n=result.model.config.latent_dim, axis=1)
|
|
527
|
+
else:
|
|
528
|
+
raise ValueError(
|
|
529
|
+
f"Your ML task {task} is not supported. Please use Latent, UMAP, PCA or RandomFeature."
|
|
530
|
+
)
|
|
531
|
+
|
|
532
|
+
return df
|