autoencodix 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of autoencodix might be problematic. Click here for more details.

Files changed (91) hide show
  1. autoencodix/__init__.py +27 -0
  2. autoencodix/base/.ipynb_checkpoints/_base_pipeline-checkpoint.py +1274 -0
  3. autoencodix/base/.ipynb_checkpoints/_base_trainer-checkpoint.py +247 -0
  4. autoencodix/base/__init__.py +21 -0
  5. autoencodix/base/_base_autoencoder.py +138 -0
  6. autoencodix/base/_base_dataset.py +103 -0
  7. autoencodix/base/_base_evaluator.py +34 -0
  8. autoencodix/base/_base_loss.py +311 -0
  9. autoencodix/base/_base_pipeline.py +1355 -0
  10. autoencodix/base/_base_predictor.py +6 -0
  11. autoencodix/base/_base_preprocessor.py +1033 -0
  12. autoencodix/base/_base_trainer.py +259 -0
  13. autoencodix/base/_base_visualizer.py +578 -0
  14. autoencodix/base/sampleconfig.yaml +132 -0
  15. autoencodix/configs/__init__.py +21 -0
  16. autoencodix/configs/default_config.py +652 -0
  17. autoencodix/configs/disentanglix_config.py +15 -0
  18. autoencodix/configs/maskix_config.py +39 -0
  19. autoencodix/configs/ontix_config.py +35 -0
  20. autoencodix/configs/stackix_config.py +33 -0
  21. autoencodix/configs/vanillix_config.py +18 -0
  22. autoencodix/configs/varix_config.py +15 -0
  23. autoencodix/configs/xmodalix_config.py +43 -0
  24. autoencodix/disentanglix.py +72 -0
  25. autoencodix/evaluate/.ipynb_checkpoints/evaluate-checkpoint.py +532 -0
  26. autoencodix/evaluate/__init__.py +4 -0
  27. autoencodix/evaluate/_general_evaluator.py +730 -0
  28. autoencodix/evaluate/_xmodalix_evaluator.py +376 -0
  29. autoencodix/imagix.py +76 -0
  30. autoencodix/losses/disentanglix_loss.py +215 -0
  31. autoencodix/losses/maskix_loss.py +94 -0
  32. autoencodix/losses/vanillix_loss.py +28 -0
  33. autoencodix/losses/varix_loss.py +102 -0
  34. autoencodix/losses/xmodal_loss.py +304 -0
  35. autoencodix/maskix.py +108 -0
  36. autoencodix/modeling/.ipynb_checkpoints/__init__-checkpoint.py +16 -0
  37. autoencodix/modeling/.ipynb_checkpoints/_captum_forward-checkpoint.py +16 -0
  38. autoencodix/modeling/__init__.py +20 -0
  39. autoencodix/modeling/_captum_forward.py +20 -0
  40. autoencodix/modeling/_classifier.py +45 -0
  41. autoencodix/modeling/_imagevae_architecture.py +296 -0
  42. autoencodix/modeling/_imgfast_architecture.py +288 -0
  43. autoencodix/modeling/_layer_factory.py +71 -0
  44. autoencodix/modeling/_maskix_architecture.py +190 -0
  45. autoencodix/modeling/_ontix_architecture.py +305 -0
  46. autoencodix/modeling/_vanillix_architecture.py +136 -0
  47. autoencodix/modeling/_varix_architecture.py +192 -0
  48. autoencodix/notebooks/__init__.py +0 -0
  49. autoencodix/ontix.py +136 -0
  50. autoencodix/py.typed +0 -0
  51. autoencodix/stackix.py +122 -0
  52. autoencodix/trainers/__init__.py +6 -0
  53. autoencodix/trainers/_general_trainer.py +525 -0
  54. autoencodix/trainers/_maskix_trainer.py +116 -0
  55. autoencodix/trainers/_ontix_trainer.py +77 -0
  56. autoencodix/trainers/_stackix_orchestrator.py +522 -0
  57. autoencodix/trainers/_stackix_trainer.py +209 -0
  58. autoencodix/trainers/_xmodal_trainer.py +1006 -0
  59. autoencodix/utils/.ipynb_checkpoints/_explainer-checkpoint.py +118 -0
  60. autoencodix/utils/.ipynb_checkpoints/_model_output-checkpoint.py +30 -0
  61. autoencodix/utils/.ipynb_checkpoints/_utils-checkpoint.py +626 -0
  62. autoencodix/utils/.ipynb_checkpoints/adata_converter-checkpoint.py +99 -0
  63. autoencodix/utils/.ipynb_checkpoints/feature_importance-checkpoint.py +823 -0
  64. autoencodix/utils/__init__.py +22 -0
  65. autoencodix/utils/_annealer.py +193 -0
  66. autoencodix/utils/_bulkreader.py +232 -0
  67. autoencodix/utils/_explainer.py +246 -0
  68. autoencodix/utils/_imgreader.py +404 -0
  69. autoencodix/utils/_internals.py +23 -0
  70. autoencodix/utils/_llm_explainer.py +318 -0
  71. autoencodix/utils/_losses.py +20 -0
  72. autoencodix/utils/_model_output.py +30 -0
  73. autoencodix/utils/_result.py +393 -0
  74. autoencodix/utils/_screader.py +76 -0
  75. autoencodix/utils/_traindynamics.py +187 -0
  76. autoencodix/utils/_utils.py +690 -0
  77. autoencodix/utils/adata_converter.py +111 -0
  78. autoencodix/utils/example_data.py +398 -0
  79. autoencodix/utils/feature_importance.py +829 -0
  80. autoencodix/utils/prompts.py +21 -0
  81. autoencodix/vanillix.py +141 -0
  82. autoencodix/varix.py +80 -0
  83. autoencodix/visualize/__init__.py +5 -0
  84. autoencodix/visualize/_general_visualizer.py +913 -0
  85. autoencodix/visualize/_imagix_visualizer.py +111 -0
  86. autoencodix/visualize/_xmodal_visualizer.py +841 -0
  87. autoencodix/visualize/visualize.py +1214 -0
  88. autoencodix/xmodalix.py +318 -0
  89. autoencodix-0.2.0.dist-info/METADATA +143 -0
  90. autoencodix-0.2.0.dist-info/RECORD +91 -0
  91. autoencodix-0.2.0.dist-info/WHEEL +4 -0
@@ -0,0 +1,730 @@
1
+ from typing import Union, no_type_check
2
+ import warnings
3
+
4
+ import numpy as np
5
+ import pandas as pd
6
+ import sklearn
7
+ from sklearn import linear_model
8
+ from sklearn.model_selection import cross_validate
9
+ from sklearn.decomposition import PCA
10
+ from sklearn.metrics import get_scorer
11
+
12
+ from umap import UMAP
13
+ from sklearn.manifold import TSNE
14
+ from sklearn.base import ClassifierMixin, RegressorMixin
15
+
16
+
17
+ from autoencodix.utils._result import Result
18
+ from autoencodix.data._datasetcontainer import DatasetContainer
19
+ from autoencodix.base._base_evaluator import BaseEvaluator
20
+ from autoencodix.base._base_visualizer import BaseVisualizer
21
+
22
+ sklearn.set_config(enable_metadata_routing=True)
23
+
24
+
25
+ class GeneralEvaluator(BaseEvaluator):
26
+ def __init__(self):
27
+ # super().__init__()
28
+ pass
29
+
30
+ @no_type_check
31
+ def evaluate(
32
+ self,
33
+ datasets: DatasetContainer,
34
+ result: Result,
35
+ ml_model_class: ClassifierMixin = linear_model.LogisticRegression(
36
+ max_iter=1000
37
+ ), # Default is sklearn LogisticRegression
38
+ ml_model_regression: RegressorMixin = linear_model.LinearRegression(), # Default is sklearn LinearRegression
39
+ params: Union[
40
+ list, str
41
+ ] = "all", # No default? ... or all params in annotation?
42
+ metric_class: str = "roc_auc_ovo", # Default is 'roc_auc_ovo' via https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-string-names
43
+ metric_regression: str = "r2", # Default is 'r2'
44
+ reference_methods: list = [], # Default [], Options are "PCA", "UMAP", "TSNE", "RandomFeature"
45
+ reference_reducer: dict = {}, # Option to provide pre-fitted reducer objects for PCA, UMAP or TSNE, e.g. {"PCA": pca_reducer, "UMAP": umap_reducer, "TSNE": tsne_reducer}
46
+ split_type: str = "use-split", # Default is "use-split", other options: "CV-5", ... "LOOCV"?
47
+ n_downsample: Union[
48
+ int, None
49
+ ] = 10000, # Default is 10000, if provided downsample to this number of samples for faster evaluation. Set to None to disable downsampling.
50
+ top_k_classes: Union[
51
+ int, None
52
+ ] = 20, # Default is 20, if provided restrict classification tasks to top k classes, others combined into "other"
53
+ exclude_classes: Union[
54
+ list, None
55
+ ] = None, # Default is None, if provided exclude these classes from evaluation
56
+ ) -> Result:
57
+ """Evaluates the performance of machine learning models on various feature representations and clinical parameters.
58
+
59
+ This method performs classification or regression tasks using specified machine learning models on different feature sets (e.g., latent space, PCA, UMAP, TSNE, RandomFeature) and clinical annotation parameters. It supports multiple evaluation strategies, including pre-defined train/valid/test splits, k-fold cross-validation, and leave-one-out cross-validation. The results are aggregated and stored in the provided `result` object.
60
+ - Samples with missing annotation values for a given parameter are excluded from the corresponding evaluation.
61
+ - For "RandomFeature", five random feature sets are evaluated.
62
+ - The method appends results to any existing `embedding_evaluation` in the result object.
63
+
64
+ Args:
65
+ datasets: A DatasetContainer containing train, valid, and test datasets, each with `sample_ids` and `metadata` (either a DataFrame or a dictionary with a 'paired' key for clinical annotations).
66
+ result: An Result object to store the evaluation results. Should have an `embedding_evaluation` attribute which updated (typically a DataFrame).
67
+ ml_model_class: The scikit-learn classifier to use for classification tasks (default: `sklearn.linear_model.LogisticRegression()`).
68
+ ml_model_regression: The scikit-learn regressor to use for regression tasks (default: `sklearn.linear_model.LinearRegression()`).
69
+ params:List of clinical annotation columns to evaluate, or "all" to use all columns (default: "all").
70
+ metric_class: Scoring metric for classification tasks (default: "roc_auc_ovo").
71
+ metric_regression: Scoring metric for regression tasks (default: "r2").
72
+ reference_methods: List of feature representations to evaluate (e.g., "PCA", "UMAP", "TSNE", "RandomFeature"). "Latent" is always included (default: []).
73
+ reference_reducer: Optional dictionary of pre-fitted dimensionality reduction objects for PCA, UMAP, or TSNE to ensure consistent transformations across runs (default: {}).
74
+ split_type: which split to use
75
+ use-split" for pre-defined splits, "CV-N" for N-fold cross-validation, or "LOOCV" for leave-one-out cross-validation (default: "use-split").
76
+ n_downsample: If provided, downsample the data to this number of samples for faster evaluation. Default is 10000. Set to None to disable downsampling.
77
+ top_k_classes: If provided, restrict classification tasks to the top k classes, combining others into "other" (default: 20).
78
+ exclude_classes: List of classes to exclude from evaluation (default: None).
79
+ Returns:
80
+ The updated result object with evaluation results stored in `embedding_evaluation`.
81
+ Raises
82
+ ValueError: If required annotation data is missing or improperly formatted, or if an unsupported split type is specified.
83
+
84
+ """
85
+
86
+ already_warned = False
87
+
88
+ df_results = pd.DataFrame()
89
+
90
+ reference_methods.append("Latent")
91
+
92
+ reference_methods = self._expand_reference_methods(
93
+ reference_methods=reference_methods, result=result
94
+ )
95
+
96
+ ## Overwrite original datasets with new_datasets if available after predict with other data
97
+ if datasets is None:
98
+ datasets = DatasetContainer()
99
+
100
+ if bool(result.new_datasets.train):
101
+ datasets.train = result.new_datasets.train
102
+ if bool(result.new_datasets.valid):
103
+ datasets.valid = result.new_datasets.valid
104
+ if bool(result.new_datasets.test):
105
+ datasets.test = result.new_datasets.test
106
+
107
+ if not bool(datasets.train or datasets.valid or datasets.test):
108
+ raise ValueError(
109
+ "No datasets found in result object. Please run predict with new data or save/load with all datasets by using save_all=True."
110
+ )
111
+ elif split_type == "use-split" and not bool(datasets.train):
112
+ warnings.warn(
113
+ "Warning: No train split found in result datasets for 'use-split' evaluation. ML model cannot be trained without a train split. Switch to cross-validation (CV-5) instead."
114
+ )
115
+ split_type = "CV-5"
116
+
117
+ for task in reference_methods:
118
+ print(f"Perform ML task with feature df: {task}")
119
+
120
+ # clin_data = self._get_clin_data(datasets)
121
+ clin_data = BaseVisualizer._collect_all_metadata(result=result)
122
+
123
+ if split_type == "use-split":
124
+ # Pandas dataframe with sample_ids and split information
125
+ sample_split = pd.DataFrame(columns=["SAMPLE_ID", "SPLIT"])
126
+
127
+ if datasets.train is not None:
128
+ if hasattr(datasets.train, "paired_sample_ids"):
129
+ if datasets.train.paired_sample_ids is not None:
130
+ sample_ids = datasets.train.paired_sample_ids
131
+ else:
132
+ sample_ids = datasets.train.sample_ids
133
+ sample_split_temp = dict(
134
+ sample_split,
135
+ **{
136
+ "SAMPLE_ID": sample_ids,
137
+ "SPLIT": ["train"] * len(sample_ids),
138
+ },
139
+ )
140
+ sample_split = pd.concat(
141
+ [sample_split, pd.DataFrame(sample_split_temp)],
142
+ axis=0,
143
+ ignore_index=True,
144
+ )
145
+ # else:
146
+ # raise ValueError(
147
+ # "No training data found. Please provide a valid training dataset."
148
+ # )
149
+ if datasets.valid is not None:
150
+ if hasattr(datasets.valid, "paired_sample_ids"):
151
+ if datasets.valid.paired_sample_ids is not None:
152
+ sample_ids = datasets.valid.paired_sample_ids
153
+ else:
154
+ sample_ids = datasets.valid.sample_ids
155
+ sample_split_temp = dict(
156
+ sample_split,
157
+ **{
158
+ "SAMPLE_ID": sample_ids,
159
+ "SPLIT": ["valid"] * len(sample_ids),
160
+ },
161
+ )
162
+ sample_split = pd.concat(
163
+ [sample_split, pd.DataFrame(sample_split_temp)],
164
+ axis=0,
165
+ ignore_index=True,
166
+ )
167
+ if datasets.test is not None:
168
+ if hasattr(datasets.test, "paired_sample_ids"):
169
+ if datasets.test.paired_sample_ids is not None:
170
+ sample_ids = datasets.test.paired_sample_ids
171
+ else:
172
+ sample_ids = datasets.test.sample_ids
173
+ sample_split_temp = dict(
174
+ sample_split,
175
+ **{
176
+ "SAMPLE_ID": sample_ids,
177
+ "SPLIT": ["test"] * len(sample_ids),
178
+ },
179
+ )
180
+ sample_split = pd.concat(
181
+ [sample_split, pd.DataFrame(sample_split_temp)],
182
+ axis=0,
183
+ ignore_index=True,
184
+ )
185
+
186
+ sample_split = sample_split.set_index("SAMPLE_ID", drop=False)
187
+
188
+ ## df -> task
189
+ subtask = [task]
190
+ if "RandomFeature" in task:
191
+ subtask = [task + "_R" + str(x) for x in range(1, 6)]
192
+ for sub in subtask:
193
+ print(sub)
194
+ # if is_modalix:
195
+ # modality = task.split("_$_")[1]
196
+ # task_xmodal = task.split("_$_")[0]
197
+
198
+ # df = self._load_input_for_ml_xmodal(task_xmodal, datasets, result, modality=modality)
199
+ # else:
200
+ df = self._load_input_for_ml(
201
+ task,
202
+ datasets,
203
+ result,
204
+ n_downsample,
205
+ reference_reducer=reference_reducer,
206
+ )
207
+
208
+ if params == "all":
209
+ params = clin_data.columns.tolist()
210
+
211
+ for task_param in params:
212
+ # if "Latent" in task:
213
+ print(f"Perform ML task for target parameter: {task_param}")
214
+ ## Check if classification or regression task
215
+ ml_type = self._get_ml_type(clin_data, task_param)
216
+
217
+ if pd.isna(clin_data[task_param]).sum() > 0:
218
+ # if pd.isna(clin_data[task_param]).values.any():
219
+ if not already_warned:
220
+ print(
221
+ "There are NA values in the annotation file. Samples with missing data will be removed for ML task evaluation."
222
+ )
223
+ already_warned = True
224
+ # logger.warning(clin_data.loc[pd.isna(clin_data[task_param]), task_param])
225
+
226
+ samples_nonna = clin_data.loc[
227
+ pd.notna(clin_data[task_param]), task_param
228
+ ].index
229
+ # print(df)
230
+ df = df.loc[samples_nonna.intersection(df.index), :]
231
+ if split_type == "use-split":
232
+ sample_split = sample_split.loc[
233
+ samples_nonna.intersection(sample_split.index), :
234
+ ]
235
+ # print(sample_split)
236
+
237
+ if n_downsample is not None:
238
+ if df.shape[0] > n_downsample:
239
+ sample_idx = np.random.choice(
240
+ df.shape[0], n_downsample, replace=False
241
+ )
242
+ df = df.iloc[sample_idx]
243
+ if split_type == "use-split":
244
+ sample_split = sample_split.loc[df.index, :]
245
+
246
+ if ml_type == "classification":
247
+ metric = metric_class
248
+ sklearn_ml = ml_model_class
249
+
250
+ if ml_type == "regression":
251
+ metric = metric_regression
252
+ sklearn_ml = ml_model_regression
253
+
254
+ if split_type == "use-split":
255
+ # print("Sample Split:")
256
+ # print(sample_split)
257
+ # print("Latent:")
258
+ # print(df)
259
+ results = self._single_ml_presplit(
260
+ sample_split=sample_split,
261
+ df=df,
262
+ clin_data=clin_data,
263
+ task_param=task_param,
264
+ sklearn_ml=sklearn_ml,
265
+ metric=metric,
266
+ ml_type=ml_type,
267
+ top_k_classes=top_k_classes,
268
+ )
269
+ elif split_type.startswith("CV-"):
270
+ cv_folds = int(split_type.split("-")[1])
271
+
272
+ results = self._single_ml(
273
+ df=df,
274
+ clin_data=clin_data,
275
+ task_param=task_param,
276
+ sklearn_ml=sklearn_ml,
277
+ metric=metric,
278
+ ml_type=ml_type,
279
+ cv_folds=cv_folds,
280
+ top_k_classes=top_k_classes,
281
+ exclude_classes=exclude_classes,
282
+ )
283
+ elif split_type == "LOOCV":
284
+ # Leave One Out Cross Validation
285
+ results = self._single_ml(
286
+ df=df,
287
+ clin_data=clin_data,
288
+ task_param=task_param,
289
+ sklearn_ml=sklearn_ml,
290
+ metric=metric,
291
+ ml_type=ml_type,
292
+ cv_folds=len(df),
293
+ top_k_classes=top_k_classes,
294
+ exclude_classes=exclude_classes,
295
+ )
296
+ else:
297
+ raise ValueError(
298
+ f"Your split type {split_type} is not supported. Please use 'use-split', 'CV-5', 'LOOCV' or 'CV-N'."
299
+ )
300
+ results = self._enrich_results(
301
+ results=results,
302
+ sklearn_ml=sklearn_ml,
303
+ ml_type=ml_type,
304
+ task=task,
305
+ sub=sub,
306
+ )
307
+
308
+ df_results = pd.concat([df_results, results])
309
+
310
+ ## Check if embedding_evaluation is empty
311
+ if (
312
+ hasattr(result, "embedding_evaluation")
313
+ and len(result.embedding_evaluation) == 0
314
+ ):
315
+ result.embedding_evaluation = df_results
316
+ else:
317
+ # merge with existing results
318
+ result.embedding_evaluation = pd.concat(
319
+ [result.embedding_evaluation, df_results], axis=0
320
+ )
321
+
322
+ return result
323
+
324
+ @staticmethod
325
+ def _single_ml(
326
+ df: pd.DataFrame,
327
+ clin_data: pd.DataFrame,
328
+ task_param: str,
329
+ sklearn_ml: Union[ClassifierMixin, RegressorMixin],
330
+ metric: str,
331
+ ml_type: str,
332
+ cv_folds: int = 5,
333
+ top_k_classes: Union[int, None] = 20,
334
+ exclude_classes: Union[list, None] = None,
335
+ ):
336
+ """Function learns on the given data frame df and label data the provided sklearn model.
337
+
338
+ Cross validation is performed according to the config and scores are returned as output as specified by metrics
339
+
340
+ Args:
341
+ df: Dataframe with input data
342
+ clin_data: Dataframe with label data
343
+ task_param: Column name with label data
344
+ sklearn_ml: Sklearn ML module specifying the ML algorithm
345
+ metric: string specifying the metric to be calculated by cross validation
346
+ ml_type: string specifying if this is a classification or regression task, used to determine which sklearn model and metric to use
347
+ cv_folds: Number of cross validation folds
348
+ top_k_classes: Number of top classes to keep, others combined into "other"
349
+ exclude_classes: List of classes to exclude from evaluation (default: None)
350
+ Returns:
351
+ score_df: data frame containing metrics (scores) for all CV runs (long format)
352
+
353
+ """
354
+
355
+ # X -> df
356
+ # Y -> task_param
357
+ y: Union[pd.Series, pd.DataFrame] = clin_data.loc[df.index, task_param]
358
+ if exclude_classes is not None:
359
+ df = df[~y.isin(exclude_classes)]
360
+ y = y[~y.isin(exclude_classes)]
361
+ score_df = dict()
362
+ # check if classification
363
+ if ml_type == "classification":
364
+ # Remove empty classes from y
365
+ if y.dtype.name == "category":
366
+ y = y.cat.remove_unused_categories()
367
+
368
+ ## Cross Validation
369
+ if len(y.unique()) > 1: # ty: ignore
370
+ # Check that more samples per class than cv_folds
371
+ # print(y.value_counts())
372
+ if ml_type == "classification":
373
+ min_class_count = y.value_counts().min() # ty: ignore
374
+ if min_class_count < cv_folds:
375
+ # Combine all classes with less than cv_folds samples into one class "other"
376
+ warnings.warn(
377
+ f"Warning: For task parameter {task_param}, some classes have less samples ({min_class_count}) than the number of CV folds ({cv_folds}). Combining these classes into one class 'other' for evaluation."
378
+ )
379
+ y = y.apply(
380
+ lambda x: (
381
+ x
382
+ if clin_data[task_param].value_counts().loc[x] >= cv_folds
383
+ else "other"
384
+ )
385
+ )
386
+ # Restrict number of classes to top k classes
387
+ if top_k_classes is not None:
388
+ if len(y.unique()) > top_k_classes:
389
+ top_k_classes_list = (
390
+ y.value_counts().nlargest(top_k_classes).index
391
+ )
392
+ y = y.apply(lambda x: x if x in top_k_classes_list else "other")
393
+ scores = cross_validate(
394
+ sklearn_ml,
395
+ df,
396
+ y,
397
+ cv=cv_folds,
398
+ scoring=metric,
399
+ return_train_score=True,
400
+ n_jobs=-1,
401
+ )
402
+
403
+ # Output
404
+
405
+ # Output Format
406
+ # CV_RUN | SCORE_SPLIT | TASK_PARAM | METRIC | VALUE
407
+
408
+ score_df["cv_run"] = list()
409
+ score_df["score_split"] = list()
410
+ score_df["CLINIC_PARAM"] = list()
411
+ score_df["metric"] = list()
412
+ score_df["value"] = list()
413
+
414
+ cv_runs = ["CV_" + str(x) for x in range(1, cv_folds + 1)]
415
+ task_param_cv = [task_param for x in range(1, cv_folds + 1)]
416
+
417
+ for m in scores:
418
+ if m.split("_")[0] == "test" or m.split("_")[0] == "train":
419
+ split_cv = [m.split("_")[0] for x in range(1, cv_folds + 1)]
420
+ metric_cv = [metric for x in range(1, cv_folds + 1)]
421
+
422
+ score_df["cv_run"].extend(cv_runs)
423
+ score_df["score_split"].extend(split_cv)
424
+ score_df["CLINIC_PARAM"].extend(task_param_cv)
425
+ score_df["metric"].extend(metric_cv)
426
+ score_df["value"].extend(scores[m])
427
+
428
+ return pd.DataFrame(score_df)
429
+
430
+ def _enrich_results(
431
+ self,
432
+ results: pd.DataFrame,
433
+ sklearn_ml: Union[ClassifierMixin, RegressorMixin],
434
+ ml_type: str,
435
+ task: str,
436
+ sub: str,
437
+ ) -> pd.DataFrame:
438
+ res_ml_alg = [str(sklearn_ml).split("(")[0] for x in range(0, results.shape[0])]
439
+ res_ml_type = [ml_type for x in range(0, results.shape[0])]
440
+ res_ml_task = [task for x in range(0, results.shape[0])]
441
+ res_ml_subtask = [sub for x in range(0, results.shape[0])]
442
+
443
+ results["ML_ALG"] = res_ml_alg
444
+ results["ML_TYPE"] = res_ml_type
445
+ # if is_modalix:
446
+ # results["MODALITY"] = [modality for x in range(0, results.shape[0])]
447
+ # results["ML_TASK"] = [task_xmodal for x in range(0, results.shape[0])]
448
+ # else:
449
+ results["ML_TASK"] = res_ml_task
450
+ results["ML_SUBTASK"] = res_ml_subtask
451
+
452
+ return results
453
+
454
+ @staticmethod
455
+ def _single_ml_presplit(
456
+ sample_split: pd.DataFrame,
457
+ df: pd.DataFrame,
458
+ clin_data: pd.DataFrame,
459
+ task_param: str,
460
+ sklearn_ml: Union[ClassifierMixin, RegressorMixin],
461
+ metric: str,
462
+ ml_type: str,
463
+ top_k_classes: Union[int, None] = 20,
464
+ ):
465
+ """Trains the provided sklearn model on the training split and evaluates it on train, valid, and test splits using the specified metric.
466
+
467
+ Args:
468
+ sample_split: DataFrame with sample IDs and their corresponding split ("train", "valid", "test").
469
+ df: DataFrame with input features, indexed by sample IDs.
470
+ clin_data: DataFrame with label/annotation data, indexed by sample IDs.
471
+ task_param: Column name in clin_data specifying the target variable.
472
+ sklearn_ml: Instantiated sklearn model to use for training and evaluation.
473
+ metric: Scoring metric compatible with sklearn's get_scorer.
474
+ ml_type: Type of machine learning task ("classification" or "regression").
475
+ top_k_classes: If provided, restrict classification tasks to the top k classes, combining others into "other" (default: 20).
476
+
477
+ Returns:
478
+ DataFrame containing evaluation scores for each split (train, valid, test) and the specified metric.
479
+
480
+ Raises
481
+ ValueError: If the provided metric is not supported by sklearn.
482
+ """
483
+ split_list = ["train", "valid", "test"]
484
+
485
+ score_df = dict()
486
+ score_df["score_split"] = list()
487
+ score_df["CLINIC_PARAM"] = list()
488
+ score_df["metric"] = list()
489
+ score_df["value"] = list()
490
+
491
+ X_train = df.loc[
492
+ sample_split.loc[sample_split.SPLIT == "train", "SAMPLE_ID"], :
493
+ ]
494
+ train_samples = [s for s in X_train.index]
495
+ Y_train = clin_data.loc[train_samples, task_param]
496
+ # train model once on training data
497
+ if len(Y_train.unique()) > 1: # ty: ignore
498
+
499
+ # Restrict number of classes to top k classes
500
+ if top_k_classes is not None and ml_type == "classification":
501
+ if len(Y_train.unique()) > top_k_classes:
502
+ top_k_classes_list = (
503
+ Y_train.value_counts().nlargest(top_k_classes).index
504
+ )
505
+ Y_train = Y_train.apply(
506
+ lambda x: x if x in top_k_classes_list else "other"
507
+ )
508
+ sklearn_ml.fit(X_train, Y_train) # ty: ignore
509
+
510
+ # eval on all splits
511
+ for split in split_list:
512
+ X = df.loc[
513
+ sample_split.loc[sample_split.SPLIT == split, "SAMPLE_ID"], :
514
+ ]
515
+ if X.shape[0] == 0:
516
+ # No samples in this split, skip
517
+ continue
518
+ samples = [s for s in X.index]
519
+ Y = clin_data.loc[samples, task_param]
520
+
521
+ # Performace on train, valid and test data split
522
+
523
+ score_df["score_split"].append(split)
524
+ score_df["CLINIC_PARAM"].append(task_param)
525
+ score_df["metric"].append(metric)
526
+ sklearn_scorer = get_scorer(metric)
527
+
528
+ if sklearn_scorer is None:
529
+ raise ValueError(
530
+ f"Your metric {metric} is not supported by sklearn. Please use a valid metric."
531
+ )
532
+
533
+ if ml_type == "classification":
534
+ if top_k_classes is not None and (
535
+ len(Y_train.unique()) > top_k_classes
536
+ ):
537
+ # Adjust Y to only contain top k classes and other as for Y_train
538
+ Y = Y.apply(lambda x: x if x in top_k_classes_list else "other")
539
+ # Check that Y has only classes which are present in Y_train
540
+ if (
541
+ len(
542
+ set(Y.unique()).difference( # ty: ignore
543
+ set(Y_train.unique()) # ty: ignore
544
+ ) # ty: ignore
545
+ ) # ty: ignore
546
+ > 0 # ty: ignore
547
+ ): # ty: ignore
548
+ print(
549
+ f"Classes in split {split} are not present in training data"
550
+ )
551
+ # Adjust Y to only contain classes present in Y_train
552
+ Y = Y[Y.isin(Y_train.unique())] # ty: ignore
553
+ # Adjust X as well
554
+ X = X.loc[Y.index, :]
555
+
556
+ if ml_type == "classification":
557
+ score_temp = sklearn_scorer(
558
+ sklearn_ml, X, Y, labels=np.sort(Y_train.unique())
559
+ )
560
+ elif ml_type == "regression":
561
+ score_temp = sklearn_scorer(sklearn_ml, X, Y)
562
+ else:
563
+ raise ValueError(
564
+ f"Your ML type {ml_type} is not supported. Please use 'classification' or 'regression'."
565
+ )
566
+ score_df["value"].append(score_temp)
567
+ else:
568
+ ## Warning that there is only one class in the training data
569
+ warnings.warn(
570
+ f"Warning: There is only one class in the training data for task parameter {task_param}. Skipping evaluation for this task."
571
+ )
572
+
573
+ return pd.DataFrame(score_df)
574
+
575
+ @staticmethod
576
+ def _get_ml_type(clin_data: pd.DataFrame, task_param: str) -> str:
577
+ """Determines the machine learning task type (classification or regression) based on the data type of a specified column in clinical data.
578
+
579
+ Args:
580
+ clin_data: The clinical data as a pandas DataFrame.
581
+ task_param: The column name in clin_data to inspect for determining the task type.
582
+
583
+ Returns:
584
+ "classification" if the first value in the specified column is a string, otherwise "regression".
585
+ """
586
+ ## Auto-Detection
587
+ if type(list(clin_data[task_param])[0]) is str:
588
+ ml_type = "classification"
589
+ elif clin_data[task_param].unique().shape[0] < 3:
590
+ ml_type = "classification"
591
+ else:
592
+ ml_type = "regression"
593
+
594
+ return ml_type
595
+
596
+ @staticmethod
597
+ def _load_input_for_ml(
598
+ task: str,
599
+ dataset: DatasetContainer,
600
+ result: Result,
601
+ n_downsample: Union[int, None] = None,
602
+ reference_reducer: dict = {},
603
+ ) -> pd.DataFrame:
604
+ """Loads and processes input data for various machine learning tasks based on the specified task type.
605
+
606
+
607
+ Task Details:
608
+ - "Latent": Concatenates latent representations from train, validation, and test splits at the final epoch.
609
+ - "UMAP": Applies UMAP dimensionality reduction to the concatenated dataset splits.
610
+ - "PCA": Applies PCA dimensionality reduction to the concatenated dataset splits.
611
+ - "TSNE": Applies t-SNE dimensionality reduction to the concatenated dataset splits.
612
+ - "RandomFeature": Randomly samples columns (features) from the concatenated dataset splits.
613
+
614
+ Args:
615
+ task: The type of ML task. Supported values are "Latent", "UMAP", "PCA", "TSNE", and "RandomFeature".
616
+ dataset: The dataset container object holding train, validation, and test splits.
617
+ result: The result object containing model configuration and methods to retrieve latent representations.
618
+ n_downsample: If provided, downsample the data to this number of samples for faster processing. Default is None (no downsampling).
619
+ reference_reducer: Optional dictionary of pre-fitted dimensionality reduction objects for PCA, UMAP, or TSNE to ensure consistent transformations across runs (default: {}).
620
+ Returns:
621
+ A DataFrame containing the processed input data suitable for the specified ML task.
622
+ Raises:
623
+ ValueError: If the provided task is not supported.
624
+ """
625
+
626
+ final_epoch = result.model.config.epochs - 1
627
+
628
+ # if task == "Latent":
629
+ # df = pd.concat(
630
+ # [
631
+ # result.get_latent_df(epoch=final_epoch, split="train"),
632
+ # result.get_latent_df(epoch=final_epoch, split="valid"),
633
+ # result.get_latent_df(epoch=-1, split="test"),
634
+ # ]
635
+ # )
636
+
637
+ if task == "Latent":
638
+ dfs = []
639
+ for split in ["train", "valid", "test"]:
640
+ df_split = result.get_latent_df(
641
+ epoch=final_epoch if split != "test" else -1, split=split
642
+ )
643
+ if df_split is not None and not df_split.empty:
644
+ if n_downsample is not None:
645
+ if df_split.shape[0] > n_downsample:
646
+ print("Downsampling data for Latent representation...")
647
+ sample_idx = np.random.choice(
648
+ df_split.shape[0], n_downsample, replace=False
649
+ )
650
+ df_split = df_split.iloc[sample_idx]
651
+ dfs.append(df_split)
652
+
653
+ df = pd.concat(dfs) if dfs else pd.DataFrame()
654
+
655
+ elif task in ["UMAP", "PCA", "TSNE", "RandomFeature"]:
656
+ dfs = []
657
+ for split_name in ["train", "valid", "test"]:
658
+ split_data = getattr(dataset, split_name, None)
659
+ if split_data is not None:
660
+ dfs.append(split_data._to_df())
661
+
662
+ if not dfs:
663
+ raise ValueError(
664
+ "No available dataset splits (train, valid, test) to process."
665
+ )
666
+
667
+ df_processed = pd.concat(dfs)
668
+
669
+ # elif task in ["UMAP", "PCA", "TSNE", "RandomFeature"]:
670
+ # if dataset.train is None:
671
+ # raise ValueError("train attribute of dataset cannot be None")
672
+ # if dataset.valid is None:
673
+ # raise ValueError("valid attribute of dataset cannot be None")
674
+ # if dataset.test is None:
675
+ # raise ValueError("test attribute of dataset cannot be None")
676
+
677
+ # df_processed = pd.concat(
678
+ # [
679
+ # dataset.train._to_df(),
680
+ # dataset.test._to_df(),
681
+ # dataset.valid._to_df(),
682
+ # ]
683
+ # )
684
+ if task == "UMAP":
685
+ if task in reference_reducer:
686
+ reducer = reference_reducer[task]
687
+ if reducer.n_components != result.model.config.latent_dim:
688
+ raise ValueError(
689
+ f"The provided UMAP reducer has n_components={reducer.n_components}, which does not match the latent dimension {result.model.config.latent_dim} specified in the model config."
690
+ )
691
+ else:
692
+ reducer = UMAP(n_components=result.model.config.latent_dim)
693
+ reducer.fit(df_processed)
694
+
695
+ df = pd.DataFrame(
696
+ reducer.transform(df_processed), index=df_processed.index
697
+ )
698
+ elif task == "PCA":
699
+ if task in reference_reducer:
700
+ reducer = reference_reducer[task]
701
+ if reducer.n_components_ != result.model.config.latent_dim:
702
+ raise ValueError(
703
+ f"The provided PCA reducer has n_components={reducer.n_components_}, which does not match the latent dimension {result.model.config.latent_dim} specified in the model config."
704
+ )
705
+ print("Using pre-fitted PCA reducer for dimensionality reduction.")
706
+ else:
707
+ reducer = PCA(n_components=result.model.config.latent_dim)
708
+ reducer.fit(df_processed)
709
+
710
+ df = pd.DataFrame(
711
+ reducer.transform(df_processed), index=df_processed.index
712
+ )
713
+ elif task == "TSNE":
714
+ if task in reference_reducer:
715
+ reducer = reference_reducer[task]
716
+ else:
717
+ reducer = TSNE(n_components=result.model.config.latent_dim)
718
+ reducer.fit(df_processed)
719
+
720
+ df = pd.DataFrame(
721
+ reducer.transform(df_processed), index=df_processed.index
722
+ )
723
+ elif task == "RandomFeature":
724
+ df = df_processed.sample(n=result.model.config.latent_dim, axis=1)
725
+ else:
726
+ raise ValueError(
727
+ f"Your ML task {task} is not supported. Please use Latent, UMAP, PCA or RandomFeature."
728
+ )
729
+
730
+ return df