autoencodix 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of autoencodix might be problematic. Click here for more details.

Files changed (91) hide show
  1. autoencodix/__init__.py +27 -0
  2. autoencodix/base/.ipynb_checkpoints/_base_pipeline-checkpoint.py +1274 -0
  3. autoencodix/base/.ipynb_checkpoints/_base_trainer-checkpoint.py +247 -0
  4. autoencodix/base/__init__.py +21 -0
  5. autoencodix/base/_base_autoencoder.py +138 -0
  6. autoencodix/base/_base_dataset.py +103 -0
  7. autoencodix/base/_base_evaluator.py +34 -0
  8. autoencodix/base/_base_loss.py +311 -0
  9. autoencodix/base/_base_pipeline.py +1355 -0
  10. autoencodix/base/_base_predictor.py +6 -0
  11. autoencodix/base/_base_preprocessor.py +1033 -0
  12. autoencodix/base/_base_trainer.py +259 -0
  13. autoencodix/base/_base_visualizer.py +578 -0
  14. autoencodix/base/sampleconfig.yaml +132 -0
  15. autoencodix/configs/__init__.py +21 -0
  16. autoencodix/configs/default_config.py +652 -0
  17. autoencodix/configs/disentanglix_config.py +15 -0
  18. autoencodix/configs/maskix_config.py +39 -0
  19. autoencodix/configs/ontix_config.py +35 -0
  20. autoencodix/configs/stackix_config.py +33 -0
  21. autoencodix/configs/vanillix_config.py +18 -0
  22. autoencodix/configs/varix_config.py +15 -0
  23. autoencodix/configs/xmodalix_config.py +43 -0
  24. autoencodix/disentanglix.py +72 -0
  25. autoencodix/evaluate/.ipynb_checkpoints/evaluate-checkpoint.py +532 -0
  26. autoencodix/evaluate/__init__.py +4 -0
  27. autoencodix/evaluate/_general_evaluator.py +730 -0
  28. autoencodix/evaluate/_xmodalix_evaluator.py +376 -0
  29. autoencodix/imagix.py +76 -0
  30. autoencodix/losses/disentanglix_loss.py +215 -0
  31. autoencodix/losses/maskix_loss.py +94 -0
  32. autoencodix/losses/vanillix_loss.py +28 -0
  33. autoencodix/losses/varix_loss.py +102 -0
  34. autoencodix/losses/xmodal_loss.py +304 -0
  35. autoencodix/maskix.py +108 -0
  36. autoencodix/modeling/.ipynb_checkpoints/__init__-checkpoint.py +16 -0
  37. autoencodix/modeling/.ipynb_checkpoints/_captum_forward-checkpoint.py +16 -0
  38. autoencodix/modeling/__init__.py +20 -0
  39. autoencodix/modeling/_captum_forward.py +20 -0
  40. autoencodix/modeling/_classifier.py +45 -0
  41. autoencodix/modeling/_imagevae_architecture.py +296 -0
  42. autoencodix/modeling/_imgfast_architecture.py +288 -0
  43. autoencodix/modeling/_layer_factory.py +71 -0
  44. autoencodix/modeling/_maskix_architecture.py +190 -0
  45. autoencodix/modeling/_ontix_architecture.py +305 -0
  46. autoencodix/modeling/_vanillix_architecture.py +136 -0
  47. autoencodix/modeling/_varix_architecture.py +192 -0
  48. autoencodix/notebooks/__init__.py +0 -0
  49. autoencodix/ontix.py +136 -0
  50. autoencodix/py.typed +0 -0
  51. autoencodix/stackix.py +122 -0
  52. autoencodix/trainers/__init__.py +6 -0
  53. autoencodix/trainers/_general_trainer.py +525 -0
  54. autoencodix/trainers/_maskix_trainer.py +116 -0
  55. autoencodix/trainers/_ontix_trainer.py +77 -0
  56. autoencodix/trainers/_stackix_orchestrator.py +522 -0
  57. autoencodix/trainers/_stackix_trainer.py +209 -0
  58. autoencodix/trainers/_xmodal_trainer.py +1006 -0
  59. autoencodix/utils/.ipynb_checkpoints/_explainer-checkpoint.py +118 -0
  60. autoencodix/utils/.ipynb_checkpoints/_model_output-checkpoint.py +30 -0
  61. autoencodix/utils/.ipynb_checkpoints/_utils-checkpoint.py +626 -0
  62. autoencodix/utils/.ipynb_checkpoints/adata_converter-checkpoint.py +99 -0
  63. autoencodix/utils/.ipynb_checkpoints/feature_importance-checkpoint.py +823 -0
  64. autoencodix/utils/__init__.py +22 -0
  65. autoencodix/utils/_annealer.py +193 -0
  66. autoencodix/utils/_bulkreader.py +232 -0
  67. autoencodix/utils/_explainer.py +246 -0
  68. autoencodix/utils/_imgreader.py +404 -0
  69. autoencodix/utils/_internals.py +23 -0
  70. autoencodix/utils/_llm_explainer.py +318 -0
  71. autoencodix/utils/_losses.py +20 -0
  72. autoencodix/utils/_model_output.py +30 -0
  73. autoencodix/utils/_result.py +393 -0
  74. autoencodix/utils/_screader.py +76 -0
  75. autoencodix/utils/_traindynamics.py +187 -0
  76. autoencodix/utils/_utils.py +690 -0
  77. autoencodix/utils/adata_converter.py +111 -0
  78. autoencodix/utils/example_data.py +398 -0
  79. autoencodix/utils/feature_importance.py +829 -0
  80. autoencodix/utils/prompts.py +21 -0
  81. autoencodix/vanillix.py +141 -0
  82. autoencodix/varix.py +80 -0
  83. autoencodix/visualize/__init__.py +5 -0
  84. autoencodix/visualize/_general_visualizer.py +913 -0
  85. autoencodix/visualize/_imagix_visualizer.py +111 -0
  86. autoencodix/visualize/_xmodal_visualizer.py +841 -0
  87. autoencodix/visualize/visualize.py +1214 -0
  88. autoencodix/xmodalix.py +318 -0
  89. autoencodix-0.2.0.dist-info/METADATA +143 -0
  90. autoencodix-0.2.0.dist-info/RECORD +91 -0
  91. autoencodix-0.2.0.dist-info/WHEEL +4 -0
@@ -0,0 +1,393 @@
1
+ from dataclasses import dataclass, field
2
+ from typing import Any, Optional, Dict, Union
3
+ from anndata import AnnData
4
+
5
+ import torch
6
+ import pandas as pd
7
+
8
+ from autoencodix.data._datasetcontainer import DatasetContainer
9
+ from autoencodix.data.datapackage import DataPackage
10
+
11
+ from mudata import MuData # type: ignore
12
+ from ._traindynamics import TrainingDynamics
13
+
14
+
15
+ @dataclass
16
+ class LossRegistry:
17
+ """Dataclass to store multiple TrainingDynamics objects for different losses.
18
+
19
+ Objective is to make the Result class extensible for multiple losses that might
20
+ be needed in the future for diffrent autoencoder architectures.
21
+
22
+ Attributes:
23
+ _losses: A dictionary to store TrainingDynamics objects for different losses.
24
+ """
25
+
26
+ _losses: Dict[str, TrainingDynamics] = field(default_factory=dict)
27
+
28
+ def __post_init__(self):
29
+ for name, data in self._losses.items():
30
+ if not isinstance(data, TrainingDynamics):
31
+ raise ValueError(
32
+ f"Expected TrainingDynamics object, got {type(data)} for loss {name}"
33
+ )
34
+
35
+ def __eq__(self, other: object) -> bool:
36
+ if not isinstance(other, LossRegistry):
37
+ return False
38
+ if self._losses.keys() != other._losses.keys():
39
+ return False
40
+ return all(self._losses[key] == other._losses[key] for key in self._losses)
41
+
42
+ def add(self, data: Dict, split: str, epoch: int) -> None:
43
+ for key, value in data.items():
44
+ if key not in self._losses:
45
+ self._losses[key] = TrainingDynamics()
46
+ self._losses[key].add(epoch=epoch, data=value, split=split)
47
+
48
+ def get(self, key: str) -> TrainingDynamics:
49
+ if key not in self._losses:
50
+ self._losses[key] = TrainingDynamics()
51
+ return self._losses[key]
52
+
53
+ def losses(self):
54
+ return self._losses.items()
55
+
56
+ def set(self, key: str, value: TrainingDynamics) -> None:
57
+ self._losses[key] = value
58
+
59
+ def keys(self):
60
+ return self._losses.keys()
61
+
62
+
63
+ @dataclass
64
+ class Result:
65
+ """A dataclass to store results from the pipeline with predefined keys.
66
+
67
+ Attributes:
68
+ latentspaces: TrainingDynamics object storing latent space representations for 'train', 'valid', and 'test' splits.
69
+ sample_ids: TrainingDynamics object storing sample identifiers for 'train', 'valid', and 'test' splits.
70
+ reconstructions: TrainingDynamics object storing reconstructed outputs for 'train', 'valid', and 'test' splits.
71
+ mus: TrainingDynamics object storing mean values of latent distributions for 'train', 'valid', and 'test' splits.
72
+ sigmas: TrainingDynamics object storing standard deviations of latent distributions for 'train', 'valid', and 'test' splits.
73
+ losses: TrainingDynamics object storing the total loss for different epochs and splits ('train', 'valid', 'test').
74
+ sub_losses: LossRegistry object (extendable) for all sublosses.
75
+ preprocessed_data: torch.Tensor containing data after preprocessing.
76
+ model: final trained torch.nn.Module model.
77
+ model_checkpoints: TrainingDynamics object storing model state at each checkpoint.
78
+ datasets: Optional[DatasetContainer] containing train, valid, and test datasets.
79
+ new_datasets: Optional[DatasetContainer] containing new train, valid, and test datasets.
80
+ adata_latent: Optional[AnnData] containing latent representations as AnnData.
81
+ final_reconstruction: Optional[Union[DataPackage, MuData]] containing final reconstruction results.
82
+ sub_results: Optional[Dict[str, Any]] containing sub-results for multi-task or multi-modal models.
83
+ sub_reconstructions: Optional[Dict[str, Any]] containing sub-reconstructions for multi-task or multi-modal models.
84
+ embedding_evaluation: pd.DataFrame containing embedding evaluation results.
85
+ """
86
+
87
+ latentspaces: TrainingDynamics = field(default_factory=TrainingDynamics)
88
+ sample_ids: TrainingDynamics = field(default_factory=TrainingDynamics)
89
+ reconstructions: TrainingDynamics = field(default_factory=TrainingDynamics)
90
+ mus: TrainingDynamics = field(default_factory=TrainingDynamics)
91
+ sigmas: TrainingDynamics = field(default_factory=TrainingDynamics)
92
+ losses: TrainingDynamics = field(default_factory=TrainingDynamics)
93
+ sub_losses: LossRegistry = field(default_factory=LossRegistry)
94
+ preprocessed_data: torch.Tensor = field(default_factory=torch.Tensor)
95
+ model: Union[Dict[str, torch.nn.Module], torch.nn.Module] = field(
96
+ default_factory=torch.nn.Module
97
+ )
98
+ model_checkpoints: TrainingDynamics = field(default_factory=TrainingDynamics)
99
+
100
+ datasets: Optional[DatasetContainer] = field(
101
+ default_factory=lambda: DatasetContainer(train=None, valid=None, test=None)
102
+ )
103
+ new_datasets: Optional[DatasetContainer] = field(
104
+ default_factory=lambda: DatasetContainer(train=None, valid=None, test=None)
105
+ )
106
+
107
+ adata_latent: Optional[AnnData] = field(default_factory=AnnData)
108
+ final_reconstruction: Optional[
109
+ Union[DataPackage, MuData] # ty: ignore[invalid-type-form]
110
+ ] = field(default=None)
111
+ sub_results: Optional[Dict[str, Any]] = field(default=None)
112
+ sub_reconstructions: Optional[Dict[str, Any]] = field(default=None)
113
+
114
+ # Embedding evaluation results
115
+ embedding_evaluation: pd.DataFrame = field(default_factory=pd.DataFrame)
116
+ embedding_attributions: pd.DataFrame = field(default_factory=pd.DataFrame)
117
+ embedding_explanations: Dict[str, str] = field(default_factory=dict)
118
+
119
+ # plots: Dict[str, Any] = field(
120
+ # default_factory=nested_dict
121
+ # ) ## Nested dictionary of plots as figure handles
122
+
123
+ def __getitem__(self, key: str) -> Any:
124
+ """Retrieve the value associated with a specific key.
125
+
126
+ Args:
127
+ key: The name of the attribute to retrieve.
128
+ Returns:
129
+ The value of the specified attribute.
130
+ Raises:
131
+ KeyError - If the key is not a valid attribute of the Results class.
132
+
133
+ """
134
+ if not hasattr(self, key):
135
+ raise KeyError(
136
+ f"Invalid key: '{key}'. Allowed keys are: {', '.join(self.__annotations__.keys())}"
137
+ )
138
+ return getattr(self, key)
139
+
140
+ def __setitem__(self, key: str, value: Any) -> None:
141
+ """Assign a value to a specific attribute.
142
+
143
+ Args:
144
+ key: The name of the attribute to set.
145
+ value: The value to assign to the attribute.
146
+ Raises:
147
+ KeyError: If the key is not a valid attribute of the Results class.
148
+
149
+ """
150
+ if not hasattr(self, key):
151
+ raise KeyError(
152
+ f"Invalid key: '{key}'. Allowed keys are: {', '.join(self.__annotations__.keys())}"
153
+ )
154
+ setattr(self, key, value)
155
+
156
+ def _is_empty_value(self, value: Any) -> bool:
157
+ """
158
+ Helper method to check if an attribute of the Result object is empty.
159
+
160
+ Parameters:
161
+ value (Any): The value to check
162
+ Returns:
163
+ bool: True if the value is empty, False otherwise
164
+
165
+ """
166
+
167
+ if isinstance(value, TrainingDynamics):
168
+ return len(value._data) == 0
169
+ elif isinstance(value, torch.Tensor):
170
+ return value.numel() == 0
171
+ elif isinstance(value, torch.nn.Module):
172
+ return sum(p.numel() for p in value.parameters()) == 0
173
+ elif isinstance(value, DatasetContainer):
174
+ return all(v is None for v in [value.train, value.valid, value.test])
175
+ elif isinstance(value, LossRegistry):
176
+ # single Nones are handled in update method (skipped)
177
+ return all(v is None for _, v in value.losses())
178
+
179
+ return False
180
+
181
+ def update(self, other: "Result") -> None:
182
+ """Update the current Result object with values from another Result object.
183
+
184
+ For TrainingDynamics, merges the data across epochs and splits and overwrites if already exists.
185
+ For all other attributes, replaces the current value with the other value.
186
+
187
+ Args:
188
+ other: The Result object to update from.
189
+ Raises:
190
+ TypeError: If the input object is not a Result instance
191
+
192
+ """
193
+ if not isinstance(other, Result):
194
+ raise TypeError(f"Expected Result object, got {type(other)}")
195
+
196
+ for field_name in self.__annotations__.keys():
197
+ current_value = getattr(self, field_name)
198
+ other_value = getattr(other, field_name)
199
+ if self._is_empty_value(other_value):
200
+ continue
201
+
202
+ # Handle TrainingDynamics - merge data
203
+ if isinstance(current_value, TrainingDynamics):
204
+ current_value = self._update_traindynamics(current_value, other_value)
205
+ # For all other types - replace with other value
206
+ if isinstance(current_value, LossRegistry):
207
+ for key, value in other_value.losses():
208
+ if value is None:
209
+ continue
210
+ if not isinstance(value, TrainingDynamics):
211
+ raise ValueError(
212
+ f"Expected TrainingDynamics object, got {type(value)}"
213
+ )
214
+ updated_dynamic = self._update_traindynamics(
215
+ current_value=current_value.get(key=key), other_value=value
216
+ )
217
+ current_value.set(key=key, value=updated_dynamic)
218
+ else:
219
+ setattr(self, field_name, other_value)
220
+
221
+ def _update_traindynamics(
222
+ self, current_value: TrainingDynamics, other_value: TrainingDynamics
223
+ ) -> TrainingDynamics:
224
+ """Update TrainingDynamics object with values from another TrainingDynamics object.
225
+
226
+ Args:
227
+ current_value: The current TrainingDynamics object to update.
228
+ other_value: The TrainingDynamics object to update from.
229
+
230
+ Returns:
231
+ Updated TrainingDynamics object.
232
+
233
+ Examples:
234
+ >>> current = TrainingDynamics()
235
+ >>> current._data = {1: {"train": np.array([1, 2, 3])},
236
+ ... 2: None}
237
+
238
+ >>> other = TrainingDynamics()
239
+ >>> other._data = {1: {"train": np.array([4, 5, 6])},
240
+ ... 2: {"train": np.array([7, 8, 9])}}
241
+ >>> # after update
242
+ >>> print(current._data)
243
+ {1: {"train": np.array([4, 5, 6])}, # updated
244
+ 2: {"train": np.array([7, 8, 9])}} # kept, because other was None
245
+
246
+ """
247
+
248
+ if current_value is None:
249
+ return other_value
250
+ if current_value._data is None:
251
+ return other_value
252
+
253
+ for epoch, split_data in other_value._data.items():
254
+ if split_data is None:
255
+ continue
256
+ if len(split_data) == 0:
257
+ continue
258
+
259
+ # If current epoch is None, it should be updated
260
+ if epoch in current_value._data and current_value._data[epoch] is None:
261
+ current_value._data[epoch] = {}
262
+ for split, data in split_data.items():
263
+ if data is None:
264
+ continue
265
+ current_value.add(epoch=epoch, data=data, split=split)
266
+ continue
267
+
268
+ if epoch not in current_value._data:
269
+ for split, data in split_data.items():
270
+ if data is None:
271
+ continue
272
+ current_value.add(epoch=epoch, data=data, split=split)
273
+ continue
274
+ # case when current epoch exists, then update all but None values
275
+ for split, value in split_data.items():
276
+ if value is not None:
277
+ current_value.add(epoch=epoch, data=value, split=split)
278
+
279
+ # Ensure ordering
280
+ current_value._data = dict(sorted(current_value._data.items()))
281
+
282
+ return current_value
283
+
284
+ def __str__(self) -> str:
285
+ """Provide a readable string representation of the Result object's public attributes.
286
+
287
+ Returns:
288
+ Formatted string showing all public attributes and their values
289
+ """
290
+ output = ["Result Object Public Attributes:", "-" * 30]
291
+
292
+ for name in self.__annotations__.keys():
293
+ if name.startswith("__"):
294
+ continue
295
+
296
+ value = getattr(self, name)
297
+ if isinstance(value, TrainingDynamics):
298
+ output.append(f"{name}: TrainingDynamics object")
299
+ elif isinstance(value, torch.nn.Module):
300
+ output.append(f"{name}: {value.__class__.__name__}")
301
+ elif isinstance(value, dict):
302
+ output.append(f"{name}: Dict with {len(value)} items")
303
+ elif isinstance(value, torch.Tensor):
304
+ output.append(f"{name}: Tensor of shape {tuple(value.shape)}")
305
+ else:
306
+ output.append(f"{name}: {value}")
307
+
308
+ return "\n".join(output)
309
+
310
+ def __repr__(self) -> str:
311
+ """Return the same representation as __str__ for consistency."""
312
+ return self.__str__()
313
+
314
+ def get_latent_df(
315
+ self, epoch: int, split: str, modality: Optional[str] = None
316
+ ) -> pd.DataFrame:
317
+ """Return latent representations as a DataFrame.
318
+
319
+ Retrieves latent vectors and their corresponding sample IDs for a given
320
+ epoch and data split. If a specific modality is provided, the results
321
+ are restricted to that modality. Column names are inferred from model
322
+ ontologies if available; otherwise, generic latent dimension labels are
323
+ used.
324
+
325
+ Args:
326
+ epoch: The epoch number to retrieve latents from.
327
+ split: The dataset split to query (e.g., "train", "valid", "test").
328
+ modality: Optional modality name to filter the latents and sample IDs.
329
+
330
+ Returns:
331
+ A DataFrame where rows correspond to samples, columns represent latent
332
+ dimensions, and the index contains sample IDs.
333
+ """
334
+ try:
335
+ latents = self.latentspaces.get(epoch=epoch, split=split)
336
+ ids = self.sample_ids.get(epoch=epoch, split=split)
337
+ if modality is not None: # for x-modalix and other multi-modal models
338
+ latents = latents[modality]
339
+ ids = ids[modality]
340
+ if hasattr(self.model, "ontologies") and self.model.ontologies is not None:
341
+ cols = list(self.model.ontologies[0].keys())
342
+ else:
343
+ cols = ["LatDim_" + str(i) for i in range(latents.shape[1])]
344
+ return pd.DataFrame(latents, index=ids, columns=cols)
345
+ except Exception as e:
346
+ import warnings
347
+
348
+ warnings.warn(
349
+ f"Could not retrieve latent representations for epoch {epoch} and split '{split}'. "
350
+ f"Returning empty DataFrame. This may be due to missing data in the Result object or incorrect keys.\n\n"
351
+ )
352
+
353
+ return pd.DataFrame()
354
+
355
+ def get_reconstructions_df(
356
+ self, epoch: int, split: str, modality: Optional[str] = None
357
+ ) -> pd.DataFrame:
358
+ """Return reconstructions as a DataFrame.
359
+
360
+ Retrieves reconstructed features and their corresponding sample IDs for a
361
+ given epoch and data split. If a specific modality is provided, the results
362
+ are restricted to that modality. Column names are based on the dataset's
363
+ feature identifiers.
364
+
365
+ Args:
366
+ epoch: The epoch number to retrieve reconstructions from.
367
+ split: The dataset split to query (e.g., "train", "valid", "test").
368
+ modality: Optional modality name to filter the reconstructions and
369
+ sample IDs.
370
+
371
+ Returns:
372
+ A DataFrame where rows correspond to samples, columns represent
373
+ reconstructed features, and the index contains sample IDs.
374
+ """
375
+ reconstructions = self.reconstructions.get(epoch=epoch, split=split)
376
+ ids = self.sample_ids.get(epoch=epoch, split=split)
377
+ if modality is not None:
378
+ reconstructions = reconstructions[modality]
379
+ ids = ids[modality]
380
+
381
+ if self.new_datasets is not None:
382
+ datasets = self.new_datasets
383
+
384
+ # cols = self.datasets.train.feature_ids
385
+ if split == "train" and datasets is not None and datasets.train is not None:
386
+ cols = datasets.train.feature_ids
387
+ elif split == "valid" and datasets is not None and datasets.valid is not None:
388
+ cols = datasets.valid.feature_ids
389
+ elif split == "test" and datasets is not None and datasets.test is not None:
390
+ cols = datasets.test.feature_ids
391
+ else:
392
+ cols = [f"Feature_{i}" for i in range(reconstructions.shape[1])]
393
+ return pd.DataFrame(reconstructions, index=ids, columns=cols)
@@ -0,0 +1,76 @@
1
+ import scanpy as sc # type: ignore
2
+ import mudata as md # type: ignore
3
+ from anndata import AnnData # type: ignore
4
+ from typing import Dict, Any, TYPE_CHECKING
5
+ from autoencodix.configs.default_config import DefaultConfig
6
+
7
+ if TYPE_CHECKING:
8
+ import mudata as md # type: ignore
9
+
10
+ MuData = md.MuData.MuData
11
+ else:
12
+ MuData = Any
13
+
14
+
15
+ class SingleCellDataReader:
16
+ """Reader for multi-modal single-cell data."""
17
+
18
+ @staticmethod
19
+ def read_data(
20
+ config: DefaultConfig,
21
+ ) -> Dict[str, MuData]: # ty: ignore[invalid-type-form]
22
+ """Read multiple single-cell modalities into MuData object(s).
23
+
24
+ Args:
25
+ config: Configuration object containing data paths and parameters.
26
+
27
+ Returns:
28
+ For non-paired translation: Dict of Dicts with {'multi_sc': DataDict} as outer dict and with modalty keys and mudata obj as inner dict.
29
+ For paired translation and non translation cases: dict with "multi_sc" as key and mudata as value
30
+ """
31
+ modalities: Dict[str, AnnData] = {}
32
+
33
+ for mod_key, mod_info in config.data_config.data_info.items():
34
+ if not mod_info.is_single_cell:
35
+ continue
36
+ adata = sc.read_h5ad(mod_info.file_path)
37
+ modalities[mod_key] = adata
38
+
39
+ # if config.requires_paired:
40
+ # mdata = md.MuData(modalities)
41
+ # common_cells = list(
42
+ # set.intersection(
43
+ # *(set(adata.obs_names) for adata in modalities.values())
44
+ # )
45
+ # )
46
+ # print(f"Number of common cells: {len(common_cells)}")
47
+ # mdata = mdata[common_cells]
48
+ # return {"multi_sc": mdata}
49
+
50
+ if config.requires_paired:
51
+ common_cells_set = set.intersection(
52
+ *(set(adata.obs_names) for adata in modalities.values())
53
+ )
54
+ common_cells_sorted = sorted(list(common_cells_set))
55
+
56
+ # Subset EACH modality individually with the sorted common cells
57
+ # This ensures each modality is aligned to the same order
58
+ aligned_modalities = {}
59
+ for mod_key, adata in modalities.items():
60
+ aligned_modalities[mod_key] = adata[common_cells_sorted].copy()
61
+ mdata = md.MuData(aligned_modalities)
62
+
63
+ print(f"Number of common cells: {len(common_cells_sorted)}")
64
+
65
+ # Clean obs_names: remove modality prefixes
66
+ cleaned_names = [
67
+ name.split(":")[-1] if ":" in name else name
68
+ for name in mdata.obs.columns
69
+ ]
70
+ mdata.obs.columns = cleaned_names
71
+
72
+ # Remove duplicate columns from obs
73
+ mdata.obs = mdata.obs.loc[:, ~mdata.obs.columns.duplicated(keep="first")]
74
+
75
+ return {"multi_sc": mdata}
76
+ return {"multi_sc": modalities}
@@ -0,0 +1,187 @@
1
+ from dataclasses import dataclass, field
2
+ from typing import Dict, Optional, Union, Any
3
+
4
+ import numpy as np
5
+
6
+
7
+ # internal check done
8
+ # write tests: done
9
+ @dataclass
10
+ class TrainingDynamics:
11
+ """Structure to store training dynamics in the form epoch -> split -> data.
12
+
13
+ Attributes:
14
+ _data: A dictionary to store numpy arrays for each epoch and split
15
+ """
16
+
17
+ _data: Dict[int, Dict[str, Union[np.ndarray, Dict]]] = field(
18
+ default_factory=dict, repr=False
19
+ )
20
+
21
+ def __eq__(self, other: object) -> bool:
22
+ if not isinstance(other, TrainingDynamics):
23
+ return False
24
+ if self._data.keys() != other._data.keys():
25
+ return False
26
+ for epoch, splits in self._data.items():
27
+ other_splits = other._data.get(epoch, {})
28
+ if set(splits.keys()) != set(other_splits.keys()):
29
+ return False
30
+ for split, data in splits.items():
31
+ other_data = other_splits.get(split, None)
32
+ if data is None and other_data is None:
33
+ continue
34
+ if isinstance(data, np.ndarray) and isinstance(other_data, np.ndarray):
35
+ if not np.array_equal(data, other_data):
36
+ return False
37
+ elif isinstance(data, dict) and isinstance(other_data, dict):
38
+ if not data == other_data: # Dict equality check
39
+ return False
40
+ elif data is None or other_data is None:
41
+ return False # One is None, the other isn't
42
+ else:
43
+ return False # Type mismatch
44
+ return True
45
+
46
+ def add(
47
+ self,
48
+ epoch: int,
49
+ data: Optional[Union[float, np.ndarray, Dict]],
50
+ split: str = "train",
51
+ ) -> None:
52
+ """
53
+ Add a numpy array for a specific epoch and split.
54
+
55
+ Parameters:
56
+ ----------
57
+ epoch : int
58
+ The epoch number.
59
+ value : np.ndarray
60
+ The numpy array to store.
61
+ split : str, optional
62
+ The data split (default: 'train').
63
+
64
+ """
65
+ if data is None:
66
+ return
67
+ if split not in ["train", "valid", "test"]:
68
+ raise KeyError(
69
+ f"Invalid split type: {split}, we only support 'train', 'valid', and 'test' splits."
70
+ )
71
+ if isinstance(data, (int, float)):
72
+ data = np.array(data)
73
+
74
+ if not isinstance(data, np.ndarray):
75
+ if not isinstance(data, Dict):
76
+ raise TypeError(
77
+ f"Expected value to be of type numpy.ndarray or Dict, got {type(data)}."
78
+ )
79
+
80
+ if epoch not in self._data:
81
+ self._data[epoch] = {}
82
+ self._data[epoch][split] = data
83
+
84
+ def get(self, epoch: Optional[int] = None, split: Optional[str] = None) -> Union[
85
+ np.ndarray,
86
+ Dict[str, np.ndarray],
87
+ Dict[int, Dict[str, np.ndarray]],
88
+ Dict[Any, Any],
89
+ ]:
90
+ """Retrieve stored numpy arrays with flexible filtering.
91
+
92
+ Args:
93
+ epoch: Specific epoch to retrieve. If None, returns data for all epochs.
94
+ split: Specific split to retrieve (e.g., 'train', 'valid', 'test').
95
+ If None, returns data for all splits.
96
+
97
+ Returns:
98
+ - If epoch is None and split is None:
99
+ Returns complete data dictionary {epoch: {split: data}}
100
+ - If epoch is None and split is provided:
101
+ Returns numpy array of values for the specified split across all epochs
102
+ - If epoch is provided and split is None:
103
+ Returns dictionary of all splits for that epoch {split: data}
104
+ - If both epoch and split are provided:
105
+ Returns numpy array for specific epoch and split
106
+
107
+ Examples:
108
+ >>> dynamics = TrainingDynamics()
109
+ >>> dynamics.add(0, np.array([0.1, 0.2]), "train")
110
+ >>> dynamics.add(1, np.array([0.2, 0.3]), "train")
111
+ >>>
112
+ >>> # Get all data
113
+ >>> dynamics.get() # Returns {0: {"train": array([0.1, 0.2])}, 1: {"train": array([0.2, 0.3])}}
114
+ >>>
115
+ >>> # Get train split across all epochs
116
+ >>> dynamics.get(split="train") # Returns array([[0.1, 0.2], [0.2, 0.3]])
117
+ >>>
118
+ >>> # Get specific epoch
119
+ >>> dynamics.get(epoch=0) # Returns {"train": array([0.1, 0.2])}
120
+ """
121
+ # Case 1: No epoch specified
122
+ if split not in ["train", "valid", "test", None]:
123
+ raise KeyError(
124
+ f"Invalid split type: {split}, we only support 'train', 'valid', and 'test' splits."
125
+ )
126
+ if len(self._data.keys()) == 0:
127
+ return {}
128
+ if epoch is None:
129
+ # Case 1a: Split specified - return array of values across epochs
130
+ if split is not None:
131
+ epochs = sorted(self._data.keys())
132
+ data = []
133
+ for e in epochs:
134
+ if split in self._data[e]:
135
+ data.append(self._data[e][split])
136
+ return np.array(data) if data else np.array([])
137
+ # Case 1b: No split specified - return complete data dictionary
138
+ return self._data
139
+
140
+ # we need the not in self._data check here, because in the predict step we save
141
+ # the model outputs with epoch -1
142
+ if epoch < 0 and epoch is not None:
143
+ # -1 equals to the highest epoch, -2 to the second highest, etc.
144
+ # for the predict case, we save the model outputs with epoch -1
145
+ # so we can index with -1 directly
146
+ epoch = (
147
+ epoch
148
+ if epoch in self._data.keys() and split == "test"
149
+ else max(self._data.keys()) + (epoch + 1)
150
+ )
151
+ if epoch >= 0 and epoch not in self._data:
152
+ if split is not None:
153
+ return np.array([])
154
+ return {}
155
+ epoch_data = self._data[epoch]
156
+
157
+ if split is None:
158
+ return epoch_data
159
+
160
+ # Case: Both epoch and split specified
161
+ return epoch_data.get(split, np.array([]))
162
+
163
+ def __getitem__(
164
+ self, key: Union[int, slice]
165
+ ) -> Union[np.ndarray, Dict[int, Dict[str, np.ndarray]], Any]:
166
+ """Allow dictionary-style and slice-based access.
167
+
168
+ Args:
169
+ key: int or slice of index to obtain
170
+ Returns:
171
+ sliced Trainingdynamic
172
+
173
+ Examples:
174
+ dynamics[100] # Get data for epoch 100.
175
+ dynamics[50:100] # Get data for epochs 50-100.
176
+ """
177
+ if isinstance(key, int):
178
+ return self.get(key)
179
+ elif isinstance(key, slice):
180
+ start = key.start or min(self._data.keys())
181
+ stop = key.stop or max(self._data.keys())
182
+ return {epoch: self.get(epoch) for epoch in range(start, stop)}
183
+ raise KeyError(f"Invalid key type: {type(key)}")
184
+
185
+ def epochs(self) -> list:
186
+ """Return all recorded epochs"""
187
+ return sorted(self._data.keys())