autoencodix 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of autoencodix might be problematic. Click here for more details.
- autoencodix/__init__.py +27 -0
- autoencodix/base/.ipynb_checkpoints/_base_pipeline-checkpoint.py +1274 -0
- autoencodix/base/.ipynb_checkpoints/_base_trainer-checkpoint.py +247 -0
- autoencodix/base/__init__.py +21 -0
- autoencodix/base/_base_autoencoder.py +138 -0
- autoencodix/base/_base_dataset.py +103 -0
- autoencodix/base/_base_evaluator.py +34 -0
- autoencodix/base/_base_loss.py +311 -0
- autoencodix/base/_base_pipeline.py +1355 -0
- autoencodix/base/_base_predictor.py +6 -0
- autoencodix/base/_base_preprocessor.py +1033 -0
- autoencodix/base/_base_trainer.py +259 -0
- autoencodix/base/_base_visualizer.py +578 -0
- autoencodix/base/sampleconfig.yaml +132 -0
- autoencodix/configs/__init__.py +21 -0
- autoencodix/configs/default_config.py +652 -0
- autoencodix/configs/disentanglix_config.py +15 -0
- autoencodix/configs/maskix_config.py +39 -0
- autoencodix/configs/ontix_config.py +35 -0
- autoencodix/configs/stackix_config.py +33 -0
- autoencodix/configs/vanillix_config.py +18 -0
- autoencodix/configs/varix_config.py +15 -0
- autoencodix/configs/xmodalix_config.py +43 -0
- autoencodix/disentanglix.py +72 -0
- autoencodix/evaluate/.ipynb_checkpoints/evaluate-checkpoint.py +532 -0
- autoencodix/evaluate/__init__.py +4 -0
- autoencodix/evaluate/_general_evaluator.py +730 -0
- autoencodix/evaluate/_xmodalix_evaluator.py +376 -0
- autoencodix/imagix.py +76 -0
- autoencodix/losses/disentanglix_loss.py +215 -0
- autoencodix/losses/maskix_loss.py +94 -0
- autoencodix/losses/vanillix_loss.py +28 -0
- autoencodix/losses/varix_loss.py +102 -0
- autoencodix/losses/xmodal_loss.py +304 -0
- autoencodix/maskix.py +108 -0
- autoencodix/modeling/.ipynb_checkpoints/__init__-checkpoint.py +16 -0
- autoencodix/modeling/.ipynb_checkpoints/_captum_forward-checkpoint.py +16 -0
- autoencodix/modeling/__init__.py +20 -0
- autoencodix/modeling/_captum_forward.py +20 -0
- autoencodix/modeling/_classifier.py +45 -0
- autoencodix/modeling/_imagevae_architecture.py +296 -0
- autoencodix/modeling/_imgfast_architecture.py +288 -0
- autoencodix/modeling/_layer_factory.py +71 -0
- autoencodix/modeling/_maskix_architecture.py +190 -0
- autoencodix/modeling/_ontix_architecture.py +305 -0
- autoencodix/modeling/_vanillix_architecture.py +136 -0
- autoencodix/modeling/_varix_architecture.py +192 -0
- autoencodix/notebooks/__init__.py +0 -0
- autoencodix/ontix.py +136 -0
- autoencodix/py.typed +0 -0
- autoencodix/stackix.py +122 -0
- autoencodix/trainers/__init__.py +6 -0
- autoencodix/trainers/_general_trainer.py +525 -0
- autoencodix/trainers/_maskix_trainer.py +116 -0
- autoencodix/trainers/_ontix_trainer.py +77 -0
- autoencodix/trainers/_stackix_orchestrator.py +522 -0
- autoencodix/trainers/_stackix_trainer.py +209 -0
- autoencodix/trainers/_xmodal_trainer.py +1006 -0
- autoencodix/utils/.ipynb_checkpoints/_explainer-checkpoint.py +118 -0
- autoencodix/utils/.ipynb_checkpoints/_model_output-checkpoint.py +30 -0
- autoencodix/utils/.ipynb_checkpoints/_utils-checkpoint.py +626 -0
- autoencodix/utils/.ipynb_checkpoints/adata_converter-checkpoint.py +99 -0
- autoencodix/utils/.ipynb_checkpoints/feature_importance-checkpoint.py +823 -0
- autoencodix/utils/__init__.py +22 -0
- autoencodix/utils/_annealer.py +193 -0
- autoencodix/utils/_bulkreader.py +232 -0
- autoencodix/utils/_explainer.py +246 -0
- autoencodix/utils/_imgreader.py +404 -0
- autoencodix/utils/_internals.py +23 -0
- autoencodix/utils/_llm_explainer.py +318 -0
- autoencodix/utils/_losses.py +20 -0
- autoencodix/utils/_model_output.py +30 -0
- autoencodix/utils/_result.py +393 -0
- autoencodix/utils/_screader.py +76 -0
- autoencodix/utils/_traindynamics.py +187 -0
- autoencodix/utils/_utils.py +690 -0
- autoencodix/utils/adata_converter.py +111 -0
- autoencodix/utils/example_data.py +398 -0
- autoencodix/utils/feature_importance.py +829 -0
- autoencodix/utils/prompts.py +21 -0
- autoencodix/vanillix.py +141 -0
- autoencodix/varix.py +80 -0
- autoencodix/visualize/__init__.py +5 -0
- autoencodix/visualize/_general_visualizer.py +913 -0
- autoencodix/visualize/_imagix_visualizer.py +111 -0
- autoencodix/visualize/_xmodal_visualizer.py +841 -0
- autoencodix/visualize/visualize.py +1214 -0
- autoencodix/xmodalix.py +318 -0
- autoencodix-0.2.0.dist-info/METADATA +143 -0
- autoencodix-0.2.0.dist-info/RECORD +91 -0
- autoencodix-0.2.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from typing import Any, Optional, Dict, Union
|
|
3
|
+
from anndata import AnnData
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from autoencodix.data._datasetcontainer import DatasetContainer
|
|
9
|
+
from autoencodix.data.datapackage import DataPackage
|
|
10
|
+
|
|
11
|
+
from mudata import MuData # type: ignore
|
|
12
|
+
from ._traindynamics import TrainingDynamics
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class LossRegistry:
|
|
17
|
+
"""Dataclass to store multiple TrainingDynamics objects for different losses.
|
|
18
|
+
|
|
19
|
+
Objective is to make the Result class extensible for multiple losses that might
|
|
20
|
+
be needed in the future for diffrent autoencoder architectures.
|
|
21
|
+
|
|
22
|
+
Attributes:
|
|
23
|
+
_losses: A dictionary to store TrainingDynamics objects for different losses.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
_losses: Dict[str, TrainingDynamics] = field(default_factory=dict)
|
|
27
|
+
|
|
28
|
+
def __post_init__(self):
|
|
29
|
+
for name, data in self._losses.items():
|
|
30
|
+
if not isinstance(data, TrainingDynamics):
|
|
31
|
+
raise ValueError(
|
|
32
|
+
f"Expected TrainingDynamics object, got {type(data)} for loss {name}"
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
def __eq__(self, other: object) -> bool:
|
|
36
|
+
if not isinstance(other, LossRegistry):
|
|
37
|
+
return False
|
|
38
|
+
if self._losses.keys() != other._losses.keys():
|
|
39
|
+
return False
|
|
40
|
+
return all(self._losses[key] == other._losses[key] for key in self._losses)
|
|
41
|
+
|
|
42
|
+
def add(self, data: Dict, split: str, epoch: int) -> None:
|
|
43
|
+
for key, value in data.items():
|
|
44
|
+
if key not in self._losses:
|
|
45
|
+
self._losses[key] = TrainingDynamics()
|
|
46
|
+
self._losses[key].add(epoch=epoch, data=value, split=split)
|
|
47
|
+
|
|
48
|
+
def get(self, key: str) -> TrainingDynamics:
|
|
49
|
+
if key not in self._losses:
|
|
50
|
+
self._losses[key] = TrainingDynamics()
|
|
51
|
+
return self._losses[key]
|
|
52
|
+
|
|
53
|
+
def losses(self):
|
|
54
|
+
return self._losses.items()
|
|
55
|
+
|
|
56
|
+
def set(self, key: str, value: TrainingDynamics) -> None:
|
|
57
|
+
self._losses[key] = value
|
|
58
|
+
|
|
59
|
+
def keys(self):
|
|
60
|
+
return self._losses.keys()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class Result:
|
|
65
|
+
"""A dataclass to store results from the pipeline with predefined keys.
|
|
66
|
+
|
|
67
|
+
Attributes:
|
|
68
|
+
latentspaces: TrainingDynamics object storing latent space representations for 'train', 'valid', and 'test' splits.
|
|
69
|
+
sample_ids: TrainingDynamics object storing sample identifiers for 'train', 'valid', and 'test' splits.
|
|
70
|
+
reconstructions: TrainingDynamics object storing reconstructed outputs for 'train', 'valid', and 'test' splits.
|
|
71
|
+
mus: TrainingDynamics object storing mean values of latent distributions for 'train', 'valid', and 'test' splits.
|
|
72
|
+
sigmas: TrainingDynamics object storing standard deviations of latent distributions for 'train', 'valid', and 'test' splits.
|
|
73
|
+
losses: TrainingDynamics object storing the total loss for different epochs and splits ('train', 'valid', 'test').
|
|
74
|
+
sub_losses: LossRegistry object (extendable) for all sublosses.
|
|
75
|
+
preprocessed_data: torch.Tensor containing data after preprocessing.
|
|
76
|
+
model: final trained torch.nn.Module model.
|
|
77
|
+
model_checkpoints: TrainingDynamics object storing model state at each checkpoint.
|
|
78
|
+
datasets: Optional[DatasetContainer] containing train, valid, and test datasets.
|
|
79
|
+
new_datasets: Optional[DatasetContainer] containing new train, valid, and test datasets.
|
|
80
|
+
adata_latent: Optional[AnnData] containing latent representations as AnnData.
|
|
81
|
+
final_reconstruction: Optional[Union[DataPackage, MuData]] containing final reconstruction results.
|
|
82
|
+
sub_results: Optional[Dict[str, Any]] containing sub-results for multi-task or multi-modal models.
|
|
83
|
+
sub_reconstructions: Optional[Dict[str, Any]] containing sub-reconstructions for multi-task or multi-modal models.
|
|
84
|
+
embedding_evaluation: pd.DataFrame containing embedding evaluation results.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
latentspaces: TrainingDynamics = field(default_factory=TrainingDynamics)
|
|
88
|
+
sample_ids: TrainingDynamics = field(default_factory=TrainingDynamics)
|
|
89
|
+
reconstructions: TrainingDynamics = field(default_factory=TrainingDynamics)
|
|
90
|
+
mus: TrainingDynamics = field(default_factory=TrainingDynamics)
|
|
91
|
+
sigmas: TrainingDynamics = field(default_factory=TrainingDynamics)
|
|
92
|
+
losses: TrainingDynamics = field(default_factory=TrainingDynamics)
|
|
93
|
+
sub_losses: LossRegistry = field(default_factory=LossRegistry)
|
|
94
|
+
preprocessed_data: torch.Tensor = field(default_factory=torch.Tensor)
|
|
95
|
+
model: Union[Dict[str, torch.nn.Module], torch.nn.Module] = field(
|
|
96
|
+
default_factory=torch.nn.Module
|
|
97
|
+
)
|
|
98
|
+
model_checkpoints: TrainingDynamics = field(default_factory=TrainingDynamics)
|
|
99
|
+
|
|
100
|
+
datasets: Optional[DatasetContainer] = field(
|
|
101
|
+
default_factory=lambda: DatasetContainer(train=None, valid=None, test=None)
|
|
102
|
+
)
|
|
103
|
+
new_datasets: Optional[DatasetContainer] = field(
|
|
104
|
+
default_factory=lambda: DatasetContainer(train=None, valid=None, test=None)
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
adata_latent: Optional[AnnData] = field(default_factory=AnnData)
|
|
108
|
+
final_reconstruction: Optional[
|
|
109
|
+
Union[DataPackage, MuData] # ty: ignore[invalid-type-form]
|
|
110
|
+
] = field(default=None)
|
|
111
|
+
sub_results: Optional[Dict[str, Any]] = field(default=None)
|
|
112
|
+
sub_reconstructions: Optional[Dict[str, Any]] = field(default=None)
|
|
113
|
+
|
|
114
|
+
# Embedding evaluation results
|
|
115
|
+
embedding_evaluation: pd.DataFrame = field(default_factory=pd.DataFrame)
|
|
116
|
+
embedding_attributions: pd.DataFrame = field(default_factory=pd.DataFrame)
|
|
117
|
+
embedding_explanations: Dict[str, str] = field(default_factory=dict)
|
|
118
|
+
|
|
119
|
+
# plots: Dict[str, Any] = field(
|
|
120
|
+
# default_factory=nested_dict
|
|
121
|
+
# ) ## Nested dictionary of plots as figure handles
|
|
122
|
+
|
|
123
|
+
def __getitem__(self, key: str) -> Any:
|
|
124
|
+
"""Retrieve the value associated with a specific key.
|
|
125
|
+
|
|
126
|
+
Args:
|
|
127
|
+
key: The name of the attribute to retrieve.
|
|
128
|
+
Returns:
|
|
129
|
+
The value of the specified attribute.
|
|
130
|
+
Raises:
|
|
131
|
+
KeyError - If the key is not a valid attribute of the Results class.
|
|
132
|
+
|
|
133
|
+
"""
|
|
134
|
+
if not hasattr(self, key):
|
|
135
|
+
raise KeyError(
|
|
136
|
+
f"Invalid key: '{key}'. Allowed keys are: {', '.join(self.__annotations__.keys())}"
|
|
137
|
+
)
|
|
138
|
+
return getattr(self, key)
|
|
139
|
+
|
|
140
|
+
def __setitem__(self, key: str, value: Any) -> None:
|
|
141
|
+
"""Assign a value to a specific attribute.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
key: The name of the attribute to set.
|
|
145
|
+
value: The value to assign to the attribute.
|
|
146
|
+
Raises:
|
|
147
|
+
KeyError: If the key is not a valid attribute of the Results class.
|
|
148
|
+
|
|
149
|
+
"""
|
|
150
|
+
if not hasattr(self, key):
|
|
151
|
+
raise KeyError(
|
|
152
|
+
f"Invalid key: '{key}'. Allowed keys are: {', '.join(self.__annotations__.keys())}"
|
|
153
|
+
)
|
|
154
|
+
setattr(self, key, value)
|
|
155
|
+
|
|
156
|
+
def _is_empty_value(self, value: Any) -> bool:
|
|
157
|
+
"""
|
|
158
|
+
Helper method to check if an attribute of the Result object is empty.
|
|
159
|
+
|
|
160
|
+
Parameters:
|
|
161
|
+
value (Any): The value to check
|
|
162
|
+
Returns:
|
|
163
|
+
bool: True if the value is empty, False otherwise
|
|
164
|
+
|
|
165
|
+
"""
|
|
166
|
+
|
|
167
|
+
if isinstance(value, TrainingDynamics):
|
|
168
|
+
return len(value._data) == 0
|
|
169
|
+
elif isinstance(value, torch.Tensor):
|
|
170
|
+
return value.numel() == 0
|
|
171
|
+
elif isinstance(value, torch.nn.Module):
|
|
172
|
+
return sum(p.numel() for p in value.parameters()) == 0
|
|
173
|
+
elif isinstance(value, DatasetContainer):
|
|
174
|
+
return all(v is None for v in [value.train, value.valid, value.test])
|
|
175
|
+
elif isinstance(value, LossRegistry):
|
|
176
|
+
# single Nones are handled in update method (skipped)
|
|
177
|
+
return all(v is None for _, v in value.losses())
|
|
178
|
+
|
|
179
|
+
return False
|
|
180
|
+
|
|
181
|
+
def update(self, other: "Result") -> None:
|
|
182
|
+
"""Update the current Result object with values from another Result object.
|
|
183
|
+
|
|
184
|
+
For TrainingDynamics, merges the data across epochs and splits and overwrites if already exists.
|
|
185
|
+
For all other attributes, replaces the current value with the other value.
|
|
186
|
+
|
|
187
|
+
Args:
|
|
188
|
+
other: The Result object to update from.
|
|
189
|
+
Raises:
|
|
190
|
+
TypeError: If the input object is not a Result instance
|
|
191
|
+
|
|
192
|
+
"""
|
|
193
|
+
if not isinstance(other, Result):
|
|
194
|
+
raise TypeError(f"Expected Result object, got {type(other)}")
|
|
195
|
+
|
|
196
|
+
for field_name in self.__annotations__.keys():
|
|
197
|
+
current_value = getattr(self, field_name)
|
|
198
|
+
other_value = getattr(other, field_name)
|
|
199
|
+
if self._is_empty_value(other_value):
|
|
200
|
+
continue
|
|
201
|
+
|
|
202
|
+
# Handle TrainingDynamics - merge data
|
|
203
|
+
if isinstance(current_value, TrainingDynamics):
|
|
204
|
+
current_value = self._update_traindynamics(current_value, other_value)
|
|
205
|
+
# For all other types - replace with other value
|
|
206
|
+
if isinstance(current_value, LossRegistry):
|
|
207
|
+
for key, value in other_value.losses():
|
|
208
|
+
if value is None:
|
|
209
|
+
continue
|
|
210
|
+
if not isinstance(value, TrainingDynamics):
|
|
211
|
+
raise ValueError(
|
|
212
|
+
f"Expected TrainingDynamics object, got {type(value)}"
|
|
213
|
+
)
|
|
214
|
+
updated_dynamic = self._update_traindynamics(
|
|
215
|
+
current_value=current_value.get(key=key), other_value=value
|
|
216
|
+
)
|
|
217
|
+
current_value.set(key=key, value=updated_dynamic)
|
|
218
|
+
else:
|
|
219
|
+
setattr(self, field_name, other_value)
|
|
220
|
+
|
|
221
|
+
def _update_traindynamics(
|
|
222
|
+
self, current_value: TrainingDynamics, other_value: TrainingDynamics
|
|
223
|
+
) -> TrainingDynamics:
|
|
224
|
+
"""Update TrainingDynamics object with values from another TrainingDynamics object.
|
|
225
|
+
|
|
226
|
+
Args:
|
|
227
|
+
current_value: The current TrainingDynamics object to update.
|
|
228
|
+
other_value: The TrainingDynamics object to update from.
|
|
229
|
+
|
|
230
|
+
Returns:
|
|
231
|
+
Updated TrainingDynamics object.
|
|
232
|
+
|
|
233
|
+
Examples:
|
|
234
|
+
>>> current = TrainingDynamics()
|
|
235
|
+
>>> current._data = {1: {"train": np.array([1, 2, 3])},
|
|
236
|
+
... 2: None}
|
|
237
|
+
|
|
238
|
+
>>> other = TrainingDynamics()
|
|
239
|
+
>>> other._data = {1: {"train": np.array([4, 5, 6])},
|
|
240
|
+
... 2: {"train": np.array([7, 8, 9])}}
|
|
241
|
+
>>> # after update
|
|
242
|
+
>>> print(current._data)
|
|
243
|
+
{1: {"train": np.array([4, 5, 6])}, # updated
|
|
244
|
+
2: {"train": np.array([7, 8, 9])}} # kept, because other was None
|
|
245
|
+
|
|
246
|
+
"""
|
|
247
|
+
|
|
248
|
+
if current_value is None:
|
|
249
|
+
return other_value
|
|
250
|
+
if current_value._data is None:
|
|
251
|
+
return other_value
|
|
252
|
+
|
|
253
|
+
for epoch, split_data in other_value._data.items():
|
|
254
|
+
if split_data is None:
|
|
255
|
+
continue
|
|
256
|
+
if len(split_data) == 0:
|
|
257
|
+
continue
|
|
258
|
+
|
|
259
|
+
# If current epoch is None, it should be updated
|
|
260
|
+
if epoch in current_value._data and current_value._data[epoch] is None:
|
|
261
|
+
current_value._data[epoch] = {}
|
|
262
|
+
for split, data in split_data.items():
|
|
263
|
+
if data is None:
|
|
264
|
+
continue
|
|
265
|
+
current_value.add(epoch=epoch, data=data, split=split)
|
|
266
|
+
continue
|
|
267
|
+
|
|
268
|
+
if epoch not in current_value._data:
|
|
269
|
+
for split, data in split_data.items():
|
|
270
|
+
if data is None:
|
|
271
|
+
continue
|
|
272
|
+
current_value.add(epoch=epoch, data=data, split=split)
|
|
273
|
+
continue
|
|
274
|
+
# case when current epoch exists, then update all but None values
|
|
275
|
+
for split, value in split_data.items():
|
|
276
|
+
if value is not None:
|
|
277
|
+
current_value.add(epoch=epoch, data=value, split=split)
|
|
278
|
+
|
|
279
|
+
# Ensure ordering
|
|
280
|
+
current_value._data = dict(sorted(current_value._data.items()))
|
|
281
|
+
|
|
282
|
+
return current_value
|
|
283
|
+
|
|
284
|
+
def __str__(self) -> str:
|
|
285
|
+
"""Provide a readable string representation of the Result object's public attributes.
|
|
286
|
+
|
|
287
|
+
Returns:
|
|
288
|
+
Formatted string showing all public attributes and their values
|
|
289
|
+
"""
|
|
290
|
+
output = ["Result Object Public Attributes:", "-" * 30]
|
|
291
|
+
|
|
292
|
+
for name in self.__annotations__.keys():
|
|
293
|
+
if name.startswith("__"):
|
|
294
|
+
continue
|
|
295
|
+
|
|
296
|
+
value = getattr(self, name)
|
|
297
|
+
if isinstance(value, TrainingDynamics):
|
|
298
|
+
output.append(f"{name}: TrainingDynamics object")
|
|
299
|
+
elif isinstance(value, torch.nn.Module):
|
|
300
|
+
output.append(f"{name}: {value.__class__.__name__}")
|
|
301
|
+
elif isinstance(value, dict):
|
|
302
|
+
output.append(f"{name}: Dict with {len(value)} items")
|
|
303
|
+
elif isinstance(value, torch.Tensor):
|
|
304
|
+
output.append(f"{name}: Tensor of shape {tuple(value.shape)}")
|
|
305
|
+
else:
|
|
306
|
+
output.append(f"{name}: {value}")
|
|
307
|
+
|
|
308
|
+
return "\n".join(output)
|
|
309
|
+
|
|
310
|
+
def __repr__(self) -> str:
|
|
311
|
+
"""Return the same representation as __str__ for consistency."""
|
|
312
|
+
return self.__str__()
|
|
313
|
+
|
|
314
|
+
def get_latent_df(
|
|
315
|
+
self, epoch: int, split: str, modality: Optional[str] = None
|
|
316
|
+
) -> pd.DataFrame:
|
|
317
|
+
"""Return latent representations as a DataFrame.
|
|
318
|
+
|
|
319
|
+
Retrieves latent vectors and their corresponding sample IDs for a given
|
|
320
|
+
epoch and data split. If a specific modality is provided, the results
|
|
321
|
+
are restricted to that modality. Column names are inferred from model
|
|
322
|
+
ontologies if available; otherwise, generic latent dimension labels are
|
|
323
|
+
used.
|
|
324
|
+
|
|
325
|
+
Args:
|
|
326
|
+
epoch: The epoch number to retrieve latents from.
|
|
327
|
+
split: The dataset split to query (e.g., "train", "valid", "test").
|
|
328
|
+
modality: Optional modality name to filter the latents and sample IDs.
|
|
329
|
+
|
|
330
|
+
Returns:
|
|
331
|
+
A DataFrame where rows correspond to samples, columns represent latent
|
|
332
|
+
dimensions, and the index contains sample IDs.
|
|
333
|
+
"""
|
|
334
|
+
try:
|
|
335
|
+
latents = self.latentspaces.get(epoch=epoch, split=split)
|
|
336
|
+
ids = self.sample_ids.get(epoch=epoch, split=split)
|
|
337
|
+
if modality is not None: # for x-modalix and other multi-modal models
|
|
338
|
+
latents = latents[modality]
|
|
339
|
+
ids = ids[modality]
|
|
340
|
+
if hasattr(self.model, "ontologies") and self.model.ontologies is not None:
|
|
341
|
+
cols = list(self.model.ontologies[0].keys())
|
|
342
|
+
else:
|
|
343
|
+
cols = ["LatDim_" + str(i) for i in range(latents.shape[1])]
|
|
344
|
+
return pd.DataFrame(latents, index=ids, columns=cols)
|
|
345
|
+
except Exception as e:
|
|
346
|
+
import warnings
|
|
347
|
+
|
|
348
|
+
warnings.warn(
|
|
349
|
+
f"Could not retrieve latent representations for epoch {epoch} and split '{split}'. "
|
|
350
|
+
f"Returning empty DataFrame. This may be due to missing data in the Result object or incorrect keys.\n\n"
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
return pd.DataFrame()
|
|
354
|
+
|
|
355
|
+
def get_reconstructions_df(
|
|
356
|
+
self, epoch: int, split: str, modality: Optional[str] = None
|
|
357
|
+
) -> pd.DataFrame:
|
|
358
|
+
"""Return reconstructions as a DataFrame.
|
|
359
|
+
|
|
360
|
+
Retrieves reconstructed features and their corresponding sample IDs for a
|
|
361
|
+
given epoch and data split. If a specific modality is provided, the results
|
|
362
|
+
are restricted to that modality. Column names are based on the dataset's
|
|
363
|
+
feature identifiers.
|
|
364
|
+
|
|
365
|
+
Args:
|
|
366
|
+
epoch: The epoch number to retrieve reconstructions from.
|
|
367
|
+
split: The dataset split to query (e.g., "train", "valid", "test").
|
|
368
|
+
modality: Optional modality name to filter the reconstructions and
|
|
369
|
+
sample IDs.
|
|
370
|
+
|
|
371
|
+
Returns:
|
|
372
|
+
A DataFrame where rows correspond to samples, columns represent
|
|
373
|
+
reconstructed features, and the index contains sample IDs.
|
|
374
|
+
"""
|
|
375
|
+
reconstructions = self.reconstructions.get(epoch=epoch, split=split)
|
|
376
|
+
ids = self.sample_ids.get(epoch=epoch, split=split)
|
|
377
|
+
if modality is not None:
|
|
378
|
+
reconstructions = reconstructions[modality]
|
|
379
|
+
ids = ids[modality]
|
|
380
|
+
|
|
381
|
+
if self.new_datasets is not None:
|
|
382
|
+
datasets = self.new_datasets
|
|
383
|
+
|
|
384
|
+
# cols = self.datasets.train.feature_ids
|
|
385
|
+
if split == "train" and datasets is not None and datasets.train is not None:
|
|
386
|
+
cols = datasets.train.feature_ids
|
|
387
|
+
elif split == "valid" and datasets is not None and datasets.valid is not None:
|
|
388
|
+
cols = datasets.valid.feature_ids
|
|
389
|
+
elif split == "test" and datasets is not None and datasets.test is not None:
|
|
390
|
+
cols = datasets.test.feature_ids
|
|
391
|
+
else:
|
|
392
|
+
cols = [f"Feature_{i}" for i in range(reconstructions.shape[1])]
|
|
393
|
+
return pd.DataFrame(reconstructions, index=ids, columns=cols)
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import scanpy as sc # type: ignore
|
|
2
|
+
import mudata as md # type: ignore
|
|
3
|
+
from anndata import AnnData # type: ignore
|
|
4
|
+
from typing import Dict, Any, TYPE_CHECKING
|
|
5
|
+
from autoencodix.configs.default_config import DefaultConfig
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
import mudata as md # type: ignore
|
|
9
|
+
|
|
10
|
+
MuData = md.MuData.MuData
|
|
11
|
+
else:
|
|
12
|
+
MuData = Any
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class SingleCellDataReader:
|
|
16
|
+
"""Reader for multi-modal single-cell data."""
|
|
17
|
+
|
|
18
|
+
@staticmethod
|
|
19
|
+
def read_data(
|
|
20
|
+
config: DefaultConfig,
|
|
21
|
+
) -> Dict[str, MuData]: # ty: ignore[invalid-type-form]
|
|
22
|
+
"""Read multiple single-cell modalities into MuData object(s).
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
config: Configuration object containing data paths and parameters.
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
For non-paired translation: Dict of Dicts with {'multi_sc': DataDict} as outer dict and with modalty keys and mudata obj as inner dict.
|
|
29
|
+
For paired translation and non translation cases: dict with "multi_sc" as key and mudata as value
|
|
30
|
+
"""
|
|
31
|
+
modalities: Dict[str, AnnData] = {}
|
|
32
|
+
|
|
33
|
+
for mod_key, mod_info in config.data_config.data_info.items():
|
|
34
|
+
if not mod_info.is_single_cell:
|
|
35
|
+
continue
|
|
36
|
+
adata = sc.read_h5ad(mod_info.file_path)
|
|
37
|
+
modalities[mod_key] = adata
|
|
38
|
+
|
|
39
|
+
# if config.requires_paired:
|
|
40
|
+
# mdata = md.MuData(modalities)
|
|
41
|
+
# common_cells = list(
|
|
42
|
+
# set.intersection(
|
|
43
|
+
# *(set(adata.obs_names) for adata in modalities.values())
|
|
44
|
+
# )
|
|
45
|
+
# )
|
|
46
|
+
# print(f"Number of common cells: {len(common_cells)}")
|
|
47
|
+
# mdata = mdata[common_cells]
|
|
48
|
+
# return {"multi_sc": mdata}
|
|
49
|
+
|
|
50
|
+
if config.requires_paired:
|
|
51
|
+
common_cells_set = set.intersection(
|
|
52
|
+
*(set(adata.obs_names) for adata in modalities.values())
|
|
53
|
+
)
|
|
54
|
+
common_cells_sorted = sorted(list(common_cells_set))
|
|
55
|
+
|
|
56
|
+
# Subset EACH modality individually with the sorted common cells
|
|
57
|
+
# This ensures each modality is aligned to the same order
|
|
58
|
+
aligned_modalities = {}
|
|
59
|
+
for mod_key, adata in modalities.items():
|
|
60
|
+
aligned_modalities[mod_key] = adata[common_cells_sorted].copy()
|
|
61
|
+
mdata = md.MuData(aligned_modalities)
|
|
62
|
+
|
|
63
|
+
print(f"Number of common cells: {len(common_cells_sorted)}")
|
|
64
|
+
|
|
65
|
+
# Clean obs_names: remove modality prefixes
|
|
66
|
+
cleaned_names = [
|
|
67
|
+
name.split(":")[-1] if ":" in name else name
|
|
68
|
+
for name in mdata.obs.columns
|
|
69
|
+
]
|
|
70
|
+
mdata.obs.columns = cleaned_names
|
|
71
|
+
|
|
72
|
+
# Remove duplicate columns from obs
|
|
73
|
+
mdata.obs = mdata.obs.loc[:, ~mdata.obs.columns.duplicated(keep="first")]
|
|
74
|
+
|
|
75
|
+
return {"multi_sc": mdata}
|
|
76
|
+
return {"multi_sc": modalities}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from typing import Dict, Optional, Union, Any
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
# internal check done
|
|
8
|
+
# write tests: done
|
|
9
|
+
@dataclass
|
|
10
|
+
class TrainingDynamics:
|
|
11
|
+
"""Structure to store training dynamics in the form epoch -> split -> data.
|
|
12
|
+
|
|
13
|
+
Attributes:
|
|
14
|
+
_data: A dictionary to store numpy arrays for each epoch and split
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
_data: Dict[int, Dict[str, Union[np.ndarray, Dict]]] = field(
|
|
18
|
+
default_factory=dict, repr=False
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
def __eq__(self, other: object) -> bool:
|
|
22
|
+
if not isinstance(other, TrainingDynamics):
|
|
23
|
+
return False
|
|
24
|
+
if self._data.keys() != other._data.keys():
|
|
25
|
+
return False
|
|
26
|
+
for epoch, splits in self._data.items():
|
|
27
|
+
other_splits = other._data.get(epoch, {})
|
|
28
|
+
if set(splits.keys()) != set(other_splits.keys()):
|
|
29
|
+
return False
|
|
30
|
+
for split, data in splits.items():
|
|
31
|
+
other_data = other_splits.get(split, None)
|
|
32
|
+
if data is None and other_data is None:
|
|
33
|
+
continue
|
|
34
|
+
if isinstance(data, np.ndarray) and isinstance(other_data, np.ndarray):
|
|
35
|
+
if not np.array_equal(data, other_data):
|
|
36
|
+
return False
|
|
37
|
+
elif isinstance(data, dict) and isinstance(other_data, dict):
|
|
38
|
+
if not data == other_data: # Dict equality check
|
|
39
|
+
return False
|
|
40
|
+
elif data is None or other_data is None:
|
|
41
|
+
return False # One is None, the other isn't
|
|
42
|
+
else:
|
|
43
|
+
return False # Type mismatch
|
|
44
|
+
return True
|
|
45
|
+
|
|
46
|
+
def add(
|
|
47
|
+
self,
|
|
48
|
+
epoch: int,
|
|
49
|
+
data: Optional[Union[float, np.ndarray, Dict]],
|
|
50
|
+
split: str = "train",
|
|
51
|
+
) -> None:
|
|
52
|
+
"""
|
|
53
|
+
Add a numpy array for a specific epoch and split.
|
|
54
|
+
|
|
55
|
+
Parameters:
|
|
56
|
+
----------
|
|
57
|
+
epoch : int
|
|
58
|
+
The epoch number.
|
|
59
|
+
value : np.ndarray
|
|
60
|
+
The numpy array to store.
|
|
61
|
+
split : str, optional
|
|
62
|
+
The data split (default: 'train').
|
|
63
|
+
|
|
64
|
+
"""
|
|
65
|
+
if data is None:
|
|
66
|
+
return
|
|
67
|
+
if split not in ["train", "valid", "test"]:
|
|
68
|
+
raise KeyError(
|
|
69
|
+
f"Invalid split type: {split}, we only support 'train', 'valid', and 'test' splits."
|
|
70
|
+
)
|
|
71
|
+
if isinstance(data, (int, float)):
|
|
72
|
+
data = np.array(data)
|
|
73
|
+
|
|
74
|
+
if not isinstance(data, np.ndarray):
|
|
75
|
+
if not isinstance(data, Dict):
|
|
76
|
+
raise TypeError(
|
|
77
|
+
f"Expected value to be of type numpy.ndarray or Dict, got {type(data)}."
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
if epoch not in self._data:
|
|
81
|
+
self._data[epoch] = {}
|
|
82
|
+
self._data[epoch][split] = data
|
|
83
|
+
|
|
84
|
+
def get(self, epoch: Optional[int] = None, split: Optional[str] = None) -> Union[
|
|
85
|
+
np.ndarray,
|
|
86
|
+
Dict[str, np.ndarray],
|
|
87
|
+
Dict[int, Dict[str, np.ndarray]],
|
|
88
|
+
Dict[Any, Any],
|
|
89
|
+
]:
|
|
90
|
+
"""Retrieve stored numpy arrays with flexible filtering.
|
|
91
|
+
|
|
92
|
+
Args:
|
|
93
|
+
epoch: Specific epoch to retrieve. If None, returns data for all epochs.
|
|
94
|
+
split: Specific split to retrieve (e.g., 'train', 'valid', 'test').
|
|
95
|
+
If None, returns data for all splits.
|
|
96
|
+
|
|
97
|
+
Returns:
|
|
98
|
+
- If epoch is None and split is None:
|
|
99
|
+
Returns complete data dictionary {epoch: {split: data}}
|
|
100
|
+
- If epoch is None and split is provided:
|
|
101
|
+
Returns numpy array of values for the specified split across all epochs
|
|
102
|
+
- If epoch is provided and split is None:
|
|
103
|
+
Returns dictionary of all splits for that epoch {split: data}
|
|
104
|
+
- If both epoch and split are provided:
|
|
105
|
+
Returns numpy array for specific epoch and split
|
|
106
|
+
|
|
107
|
+
Examples:
|
|
108
|
+
>>> dynamics = TrainingDynamics()
|
|
109
|
+
>>> dynamics.add(0, np.array([0.1, 0.2]), "train")
|
|
110
|
+
>>> dynamics.add(1, np.array([0.2, 0.3]), "train")
|
|
111
|
+
>>>
|
|
112
|
+
>>> # Get all data
|
|
113
|
+
>>> dynamics.get() # Returns {0: {"train": array([0.1, 0.2])}, 1: {"train": array([0.2, 0.3])}}
|
|
114
|
+
>>>
|
|
115
|
+
>>> # Get train split across all epochs
|
|
116
|
+
>>> dynamics.get(split="train") # Returns array([[0.1, 0.2], [0.2, 0.3]])
|
|
117
|
+
>>>
|
|
118
|
+
>>> # Get specific epoch
|
|
119
|
+
>>> dynamics.get(epoch=0) # Returns {"train": array([0.1, 0.2])}
|
|
120
|
+
"""
|
|
121
|
+
# Case 1: No epoch specified
|
|
122
|
+
if split not in ["train", "valid", "test", None]:
|
|
123
|
+
raise KeyError(
|
|
124
|
+
f"Invalid split type: {split}, we only support 'train', 'valid', and 'test' splits."
|
|
125
|
+
)
|
|
126
|
+
if len(self._data.keys()) == 0:
|
|
127
|
+
return {}
|
|
128
|
+
if epoch is None:
|
|
129
|
+
# Case 1a: Split specified - return array of values across epochs
|
|
130
|
+
if split is not None:
|
|
131
|
+
epochs = sorted(self._data.keys())
|
|
132
|
+
data = []
|
|
133
|
+
for e in epochs:
|
|
134
|
+
if split in self._data[e]:
|
|
135
|
+
data.append(self._data[e][split])
|
|
136
|
+
return np.array(data) if data else np.array([])
|
|
137
|
+
# Case 1b: No split specified - return complete data dictionary
|
|
138
|
+
return self._data
|
|
139
|
+
|
|
140
|
+
# we need the not in self._data check here, because in the predict step we save
|
|
141
|
+
# the model outputs with epoch -1
|
|
142
|
+
if epoch < 0 and epoch is not None:
|
|
143
|
+
# -1 equals to the highest epoch, -2 to the second highest, etc.
|
|
144
|
+
# for the predict case, we save the model outputs with epoch -1
|
|
145
|
+
# so we can index with -1 directly
|
|
146
|
+
epoch = (
|
|
147
|
+
epoch
|
|
148
|
+
if epoch in self._data.keys() and split == "test"
|
|
149
|
+
else max(self._data.keys()) + (epoch + 1)
|
|
150
|
+
)
|
|
151
|
+
if epoch >= 0 and epoch not in self._data:
|
|
152
|
+
if split is not None:
|
|
153
|
+
return np.array([])
|
|
154
|
+
return {}
|
|
155
|
+
epoch_data = self._data[epoch]
|
|
156
|
+
|
|
157
|
+
if split is None:
|
|
158
|
+
return epoch_data
|
|
159
|
+
|
|
160
|
+
# Case: Both epoch and split specified
|
|
161
|
+
return epoch_data.get(split, np.array([]))
|
|
162
|
+
|
|
163
|
+
def __getitem__(
|
|
164
|
+
self, key: Union[int, slice]
|
|
165
|
+
) -> Union[np.ndarray, Dict[int, Dict[str, np.ndarray]], Any]:
|
|
166
|
+
"""Allow dictionary-style and slice-based access.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
key: int or slice of index to obtain
|
|
170
|
+
Returns:
|
|
171
|
+
sliced Trainingdynamic
|
|
172
|
+
|
|
173
|
+
Examples:
|
|
174
|
+
dynamics[100] # Get data for epoch 100.
|
|
175
|
+
dynamics[50:100] # Get data for epochs 50-100.
|
|
176
|
+
"""
|
|
177
|
+
if isinstance(key, int):
|
|
178
|
+
return self.get(key)
|
|
179
|
+
elif isinstance(key, slice):
|
|
180
|
+
start = key.start or min(self._data.keys())
|
|
181
|
+
stop = key.stop or max(self._data.keys())
|
|
182
|
+
return {epoch: self.get(epoch) for epoch in range(start, stop)}
|
|
183
|
+
raise KeyError(f"Invalid key type: {type(key)}")
|
|
184
|
+
|
|
185
|
+
def epochs(self) -> list:
|
|
186
|
+
"""Return all recorded epochs"""
|
|
187
|
+
return sorted(self._data.keys())
|