autoencodix 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of autoencodix might be problematic. Click here for more details.
- autoencodix/__init__.py +27 -0
- autoencodix/base/.ipynb_checkpoints/_base_pipeline-checkpoint.py +1274 -0
- autoencodix/base/.ipynb_checkpoints/_base_trainer-checkpoint.py +247 -0
- autoencodix/base/__init__.py +21 -0
- autoencodix/base/_base_autoencoder.py +138 -0
- autoencodix/base/_base_dataset.py +103 -0
- autoencodix/base/_base_evaluator.py +34 -0
- autoencodix/base/_base_loss.py +311 -0
- autoencodix/base/_base_pipeline.py +1355 -0
- autoencodix/base/_base_predictor.py +6 -0
- autoencodix/base/_base_preprocessor.py +1033 -0
- autoencodix/base/_base_trainer.py +259 -0
- autoencodix/base/_base_visualizer.py +578 -0
- autoencodix/base/sampleconfig.yaml +132 -0
- autoencodix/configs/__init__.py +21 -0
- autoencodix/configs/default_config.py +652 -0
- autoencodix/configs/disentanglix_config.py +15 -0
- autoencodix/configs/maskix_config.py +39 -0
- autoencodix/configs/ontix_config.py +35 -0
- autoencodix/configs/stackix_config.py +33 -0
- autoencodix/configs/vanillix_config.py +18 -0
- autoencodix/configs/varix_config.py +15 -0
- autoencodix/configs/xmodalix_config.py +43 -0
- autoencodix/disentanglix.py +72 -0
- autoencodix/evaluate/.ipynb_checkpoints/evaluate-checkpoint.py +532 -0
- autoencodix/evaluate/__init__.py +4 -0
- autoencodix/evaluate/_general_evaluator.py +730 -0
- autoencodix/evaluate/_xmodalix_evaluator.py +376 -0
- autoencodix/imagix.py +76 -0
- autoencodix/losses/disentanglix_loss.py +215 -0
- autoencodix/losses/maskix_loss.py +94 -0
- autoencodix/losses/vanillix_loss.py +28 -0
- autoencodix/losses/varix_loss.py +102 -0
- autoencodix/losses/xmodal_loss.py +304 -0
- autoencodix/maskix.py +108 -0
- autoencodix/modeling/.ipynb_checkpoints/__init__-checkpoint.py +16 -0
- autoencodix/modeling/.ipynb_checkpoints/_captum_forward-checkpoint.py +16 -0
- autoencodix/modeling/__init__.py +20 -0
- autoencodix/modeling/_captum_forward.py +20 -0
- autoencodix/modeling/_classifier.py +45 -0
- autoencodix/modeling/_imagevae_architecture.py +296 -0
- autoencodix/modeling/_imgfast_architecture.py +288 -0
- autoencodix/modeling/_layer_factory.py +71 -0
- autoencodix/modeling/_maskix_architecture.py +190 -0
- autoencodix/modeling/_ontix_architecture.py +305 -0
- autoencodix/modeling/_vanillix_architecture.py +136 -0
- autoencodix/modeling/_varix_architecture.py +192 -0
- autoencodix/notebooks/__init__.py +0 -0
- autoencodix/ontix.py +136 -0
- autoencodix/py.typed +0 -0
- autoencodix/stackix.py +122 -0
- autoencodix/trainers/__init__.py +6 -0
- autoencodix/trainers/_general_trainer.py +525 -0
- autoencodix/trainers/_maskix_trainer.py +116 -0
- autoencodix/trainers/_ontix_trainer.py +77 -0
- autoencodix/trainers/_stackix_orchestrator.py +522 -0
- autoencodix/trainers/_stackix_trainer.py +209 -0
- autoencodix/trainers/_xmodal_trainer.py +1006 -0
- autoencodix/utils/.ipynb_checkpoints/_explainer-checkpoint.py +118 -0
- autoencodix/utils/.ipynb_checkpoints/_model_output-checkpoint.py +30 -0
- autoencodix/utils/.ipynb_checkpoints/_utils-checkpoint.py +626 -0
- autoencodix/utils/.ipynb_checkpoints/adata_converter-checkpoint.py +99 -0
- autoencodix/utils/.ipynb_checkpoints/feature_importance-checkpoint.py +823 -0
- autoencodix/utils/__init__.py +22 -0
- autoencodix/utils/_annealer.py +193 -0
- autoencodix/utils/_bulkreader.py +232 -0
- autoencodix/utils/_explainer.py +246 -0
- autoencodix/utils/_imgreader.py +404 -0
- autoencodix/utils/_internals.py +23 -0
- autoencodix/utils/_llm_explainer.py +318 -0
- autoencodix/utils/_losses.py +20 -0
- autoencodix/utils/_model_output.py +30 -0
- autoencodix/utils/_result.py +393 -0
- autoencodix/utils/_screader.py +76 -0
- autoencodix/utils/_traindynamics.py +187 -0
- autoencodix/utils/_utils.py +690 -0
- autoencodix/utils/adata_converter.py +111 -0
- autoencodix/utils/example_data.py +398 -0
- autoencodix/utils/feature_importance.py +829 -0
- autoencodix/utils/prompts.py +21 -0
- autoencodix/vanillix.py +141 -0
- autoencodix/varix.py +80 -0
- autoencodix/visualize/__init__.py +5 -0
- autoencodix/visualize/_general_visualizer.py +913 -0
- autoencodix/visualize/_imagix_visualizer.py +111 -0
- autoencodix/visualize/_xmodal_visualizer.py +841 -0
- autoencodix/visualize/visualize.py +1214 -0
- autoencodix/xmodalix.py +318 -0
- autoencodix-0.2.0.dist-info/METADATA +143 -0
- autoencodix-0.2.0.dist-info/RECORD +91 -0
- autoencodix-0.2.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,525 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
|
|
3
|
+
import gc
|
|
4
|
+
import os
|
|
5
|
+
import numpy as np
|
|
6
|
+
from typing import Optional, Type, Union, Tuple, Any, Dict, List
|
|
7
|
+
from collections import defaultdict
|
|
8
|
+
from torch.utils.data import DataLoader
|
|
9
|
+
|
|
10
|
+
from autoencodix.base._base_dataset import BaseDataset
|
|
11
|
+
from autoencodix.base._base_loss import BaseLoss
|
|
12
|
+
from autoencodix.base._base_autoencoder import BaseAutoencoder
|
|
13
|
+
from autoencodix.base._base_trainer import BaseTrainer
|
|
14
|
+
from autoencodix.utils._result import Result
|
|
15
|
+
from autoencodix.configs.default_config import DefaultConfig
|
|
16
|
+
from autoencodix.utils._model_output import ModelOutput
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class GeneralTrainer(BaseTrainer):
|
|
20
|
+
"""Handles general training logic for autoencoder models.
|
|
21
|
+
|
|
22
|
+
Attributes:
|
|
23
|
+
_trainset: The dataset used for training.
|
|
24
|
+
_validset: The dataset used for validation, if provided.
|
|
25
|
+
_result: An object to store and manage training results.
|
|
26
|
+
_config: Configuration object containing training hyperparameters and settings.
|
|
27
|
+
_model_type: The autoencoder model class to be trained.
|
|
28
|
+
_loss_fn: Instantiated loss function specific to the model.
|
|
29
|
+
_trainloader: DataLoader for the training dataset.
|
|
30
|
+
_validloader: DataLoader for the validation dataset, if provided.
|
|
31
|
+
_model: The instantiated model architecture.
|
|
32
|
+
_optimizer: The optimizer used for training.
|
|
33
|
+
_fabric: Lightning Fabric wrapper for device and precision management.
|
|
34
|
+
n_train: Number of training samples.
|
|
35
|
+
n_valid: Number of validation samples.
|
|
36
|
+
n_test: Number of test samples (set during prediction).
|
|
37
|
+
n_features: Number of input features.
|
|
38
|
+
latent_dim: Dimensionality of the latent space.
|
|
39
|
+
device: Device on which the model is located.
|
|
40
|
+
_n_cpus: Number of CPU cores available.
|
|
41
|
+
_reconstruction_buffer: Buffer to store reconstructions during training/validation/testing.
|
|
42
|
+
_latentspace_buffer: Buffer to store latent representations during training/validation/testing.
|
|
43
|
+
_mu_buffer: Buffer to store latent means (for VAE) during training/validation
|
|
44
|
+
_sigma_buffer: Buffer to store latent log-variances (for VAE) during training/validation/testing.
|
|
45
|
+
_sample_ids_buffer: Buffer to store sample IDs during training/validation/testing.
|
|
46
|
+
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def __init__(
|
|
50
|
+
self,
|
|
51
|
+
trainset: Optional[BaseDataset],
|
|
52
|
+
validset: Optional[BaseDataset],
|
|
53
|
+
result: Result,
|
|
54
|
+
config: DefaultConfig,
|
|
55
|
+
model_type: Type[BaseAutoencoder],
|
|
56
|
+
loss_type: Type[BaseLoss],
|
|
57
|
+
ontologies: Optional[Union[Tuple, List]] = None,
|
|
58
|
+
**kwargs,
|
|
59
|
+
):
|
|
60
|
+
"""Initializes the GeneralTrainer.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
trainset: The dataset used for training.
|
|
64
|
+
validset: The dataset used for validation, if provided.
|
|
65
|
+
result: An object to store and manage training results.
|
|
66
|
+
config: Configuration object containing training hyperparameters and settings.
|
|
67
|
+
model_type: The autoencoder model class to be trained.
|
|
68
|
+
loss_type: The loss function class to be used for training.
|
|
69
|
+
ontologies: Ontology information, if provided for Ontix
|
|
70
|
+
"""
|
|
71
|
+
self._n_cpus = os.cpu_count()
|
|
72
|
+
if self._n_cpus is None:
|
|
73
|
+
self._n_cpus = 0
|
|
74
|
+
|
|
75
|
+
super().__init__(
|
|
76
|
+
trainset, validset, result, config, model_type, loss_type, ontologies
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
self.latent_dim = config.latent_dim
|
|
80
|
+
if ontologies is not None:
|
|
81
|
+
if not hasattr(self._model, "latent_dim"):
|
|
82
|
+
raise ValueError(
|
|
83
|
+
"Model must have a 'latent_dim' attribute when ontologies are provided."
|
|
84
|
+
)
|
|
85
|
+
self.latent_dim = self._model.latent_dim
|
|
86
|
+
|
|
87
|
+
# we will set this later, in the predict method
|
|
88
|
+
self.n_test: Optional[int] = None
|
|
89
|
+
self.n_train = len(trainset) if trainset else 0
|
|
90
|
+
self.n_valid = len(validset) if validset else 0
|
|
91
|
+
self.n_features = trainset.get_input_dim() if trainset else 0
|
|
92
|
+
self.device = next(self._model.parameters()).device
|
|
93
|
+
self._init_buffers()
|
|
94
|
+
|
|
95
|
+
def _init_buffers(self, input_data: Optional[BaseDataset] = None) -> None:
|
|
96
|
+
if input_data:
|
|
97
|
+
self.n_features = input_data.get_input_dim()
|
|
98
|
+
|
|
99
|
+
def make_tensor_buffer(size: int, dim: Union[int, Tuple[int, ...]]):
|
|
100
|
+
if isinstance(dim, int):
|
|
101
|
+
if self._config.save_vram:
|
|
102
|
+
return torch.zeros(
|
|
103
|
+
(size, dim), device="cpu"
|
|
104
|
+
) # Move to CPU to avoid GPU OOM, but will be slower
|
|
105
|
+
else:
|
|
106
|
+
return torch.zeros((size, dim), device=self.device)
|
|
107
|
+
else:
|
|
108
|
+
if self._config.save_vram:
|
|
109
|
+
return torch.zeros(
|
|
110
|
+
(size, *dim), device="cpu"
|
|
111
|
+
) # Move to CPU to avoid GPU OOM, but will be slower
|
|
112
|
+
else:
|
|
113
|
+
return torch.zeros((size, *dim), device=self.device)
|
|
114
|
+
|
|
115
|
+
def make_numpy_buffer(size: int):
|
|
116
|
+
return np.empty((size,), dtype=object)
|
|
117
|
+
|
|
118
|
+
# Always create sample ID buffers
|
|
119
|
+
self._sample_ids_buffer = {
|
|
120
|
+
"train": make_numpy_buffer(self.n_train),
|
|
121
|
+
"valid": make_numpy_buffer(self.n_valid) if self.n_valid else None,
|
|
122
|
+
"test": make_numpy_buffer(self.n_test) if self.n_test else None,
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
splits = ["test"] if self._config.save_memory else ["train", "valid", "test"]
|
|
126
|
+
|
|
127
|
+
def make_split_buffers(dim):
|
|
128
|
+
return {
|
|
129
|
+
split: (
|
|
130
|
+
make_tensor_buffer(getattr(self, f"n_{split}"), dim)
|
|
131
|
+
if getattr(self, f"n_{split}", 0) and (split in splits)
|
|
132
|
+
else None
|
|
133
|
+
)
|
|
134
|
+
for split in ["train", "valid", "test"]
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
self._latentspace_buffer = make_split_buffers(self.latent_dim)
|
|
138
|
+
self._reconstruction_buffer = make_split_buffers(self.n_features)
|
|
139
|
+
self._mu_buffer = make_split_buffers(self.latent_dim)
|
|
140
|
+
self._sigma_buffer = make_split_buffers(self.latent_dim)
|
|
141
|
+
|
|
142
|
+
def _ontix_hook(self):
|
|
143
|
+
pass
|
|
144
|
+
|
|
145
|
+
def train(self, epochs_overwrite=None) -> Result:
|
|
146
|
+
"""Orchestrates training over multiple epochs, including training and validation phases.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
epochs_overwrite: If provided, overrides the number of epochs specified in the config.
|
|
150
|
+
This is only there so we can use the train method for pretraining.Any
|
|
151
|
+
Returns:
|
|
152
|
+
Result object containing training results and dynamics like latent spaces and reconstructions.
|
|
153
|
+
"""
|
|
154
|
+
epochs = self._config.epochs
|
|
155
|
+
if epochs_overwrite:
|
|
156
|
+
epochs = epochs_overwrite
|
|
157
|
+
with self._fabric.autocast():
|
|
158
|
+
for epoch in range(epochs):
|
|
159
|
+
self._init_buffers()
|
|
160
|
+
should_checkpoint: bool = self._should_checkpoint(epoch)
|
|
161
|
+
self._model.train()
|
|
162
|
+
|
|
163
|
+
epoch_loss, epoch_sub_losses = self._train_epoch(
|
|
164
|
+
should_checkpoint=should_checkpoint, epoch=epoch
|
|
165
|
+
)
|
|
166
|
+
self._log_losses(epoch, "train", epoch_loss, epoch_sub_losses)
|
|
167
|
+
|
|
168
|
+
if self._validset:
|
|
169
|
+
valid_loss, valid_sub_losses = self._validate_epoch(
|
|
170
|
+
should_checkpoint=should_checkpoint, epoch=epoch
|
|
171
|
+
)
|
|
172
|
+
self._log_losses(epoch, "valid", valid_loss, valid_sub_losses)
|
|
173
|
+
|
|
174
|
+
if should_checkpoint:
|
|
175
|
+
self._store_checkpoint(epoch)
|
|
176
|
+
|
|
177
|
+
self._result.model = next(self._model.children())
|
|
178
|
+
return self._result
|
|
179
|
+
|
|
180
|
+
def maskix_hook(self, X: torch.Tensor):
|
|
181
|
+
"""Only active when override from MaskixTrainer is used"""
|
|
182
|
+
return X
|
|
183
|
+
|
|
184
|
+
def _train_epoch(
|
|
185
|
+
self, should_checkpoint: bool, epoch: int
|
|
186
|
+
) -> Tuple[float, Dict[str, float]]:
|
|
187
|
+
"""Handles loss computation, backwards pass and checkpointing for one epoch.
|
|
188
|
+
|
|
189
|
+
Args:
|
|
190
|
+
should_checkpoint: Whether to checkpoint this epoch.
|
|
191
|
+
epoch: The current epoch number.
|
|
192
|
+
Returns:
|
|
193
|
+
total_loss: The total loss for the epoch.
|
|
194
|
+
sub_losses: A dictionary of sub-losses accumulated over the epoch.
|
|
195
|
+
"""
|
|
196
|
+
total_loss = 0.0
|
|
197
|
+
sub_losses: Dict[str, float] = defaultdict(float)
|
|
198
|
+
current_batch = 0
|
|
199
|
+
for indices, features, sample_ids in self._trainloader:
|
|
200
|
+
current_batch += 1
|
|
201
|
+
self._optimizer.zero_grad()
|
|
202
|
+
X: torch.Tensor = self.maskix_hook(features)
|
|
203
|
+
model_outputs = self._model(X)
|
|
204
|
+
loss, batch_sub_losses = self._loss_fn(
|
|
205
|
+
model_output=model_outputs,
|
|
206
|
+
targets=features,
|
|
207
|
+
corrupted_input=X, # only relevant for MaskixLoss
|
|
208
|
+
epoch=epoch,
|
|
209
|
+
n_samples=len(
|
|
210
|
+
self._trainloader.dataset
|
|
211
|
+
), # Pass n_samples for disentangled loss calculations
|
|
212
|
+
)
|
|
213
|
+
self._fabric.backward(loss)
|
|
214
|
+
self._ontix_hook()
|
|
215
|
+
self._optimizer.step()
|
|
216
|
+
|
|
217
|
+
total_loss += loss.item()
|
|
218
|
+
for k, v in batch_sub_losses.items():
|
|
219
|
+
if "_factor" not in k: # Skip factor losses
|
|
220
|
+
sub_losses[k] += v.item()
|
|
221
|
+
else:
|
|
222
|
+
sub_losses[k] = v.item()
|
|
223
|
+
|
|
224
|
+
if should_checkpoint:
|
|
225
|
+
self._capture_dynamics(model_outputs, "train", indices, sample_ids)
|
|
226
|
+
|
|
227
|
+
for k, v in sub_losses.items():
|
|
228
|
+
if "_factor" not in k:
|
|
229
|
+
sub_losses[k] = v / len(
|
|
230
|
+
self._trainloader.dataset
|
|
231
|
+
) # Average over all samples
|
|
232
|
+
total_loss = total_loss / len(
|
|
233
|
+
self._trainloader.dataset
|
|
234
|
+
) # Average over all samples
|
|
235
|
+
return total_loss, sub_losses
|
|
236
|
+
|
|
237
|
+
def _validate_epoch(
|
|
238
|
+
self, should_checkpoint: bool, epoch: int
|
|
239
|
+
) -> Tuple[float, Dict[str, float]]:
|
|
240
|
+
"""Handles validation for one epoch.
|
|
241
|
+
|
|
242
|
+
Args:
|
|
243
|
+
should_checkpoint: Whether to checkpoint this epoch.
|
|
244
|
+
epoch: The current epoch number.
|
|
245
|
+
Returns:
|
|
246
|
+
total_loss: The total loss for the epoch.
|
|
247
|
+
sub_losses: A dictionary of sub-losses accumulated over the epoch.
|
|
248
|
+
"""
|
|
249
|
+
total_loss = 0.0
|
|
250
|
+
sub_losses: Dict[str, float] = defaultdict(float)
|
|
251
|
+
self._model.eval()
|
|
252
|
+
|
|
253
|
+
with torch.no_grad():
|
|
254
|
+
for indices, features, sample_ids in self._validloader:
|
|
255
|
+
X: torch.Tensor = self.maskix_hook(features)
|
|
256
|
+
model_outputs = self._model(X)
|
|
257
|
+
loss, batch_sub_losses = self._loss_fn(
|
|
258
|
+
model_output=model_outputs,
|
|
259
|
+
targets=features,
|
|
260
|
+
corrupted_input=X,
|
|
261
|
+
epoch=epoch,
|
|
262
|
+
n_samples=len(
|
|
263
|
+
self._validloader.dataset
|
|
264
|
+
), # Pass n_samples for disentangled loss calculations
|
|
265
|
+
)
|
|
266
|
+
total_loss += loss.item()
|
|
267
|
+
for k, v in batch_sub_losses.items():
|
|
268
|
+
if "_factor" not in k: # Skip factor losses
|
|
269
|
+
sub_losses[k] += v.item()
|
|
270
|
+
else:
|
|
271
|
+
sub_losses[k] = v.item()
|
|
272
|
+
if should_checkpoint:
|
|
273
|
+
self._capture_dynamics(model_outputs, "valid", indices, sample_ids)
|
|
274
|
+
|
|
275
|
+
for k, v in sub_losses.items():
|
|
276
|
+
if "_factor" not in k:
|
|
277
|
+
sub_losses[k] = v / len(
|
|
278
|
+
self._validloader.dataset
|
|
279
|
+
) # Average over all samples
|
|
280
|
+
total_loss = total_loss / len(
|
|
281
|
+
self._validloader.dataset
|
|
282
|
+
) # Average over all samples
|
|
283
|
+
return total_loss, sub_losses
|
|
284
|
+
|
|
285
|
+
def _log_losses(
|
|
286
|
+
self, epoch: int, split: str, total_loss: float, sub_losses: Dict[str, float]
|
|
287
|
+
) -> None:
|
|
288
|
+
"""Logs the total and sub-losses for an epoch and stores them in the Result object.
|
|
289
|
+
|
|
290
|
+
Args:
|
|
291
|
+
epoch: The current epoch number.
|
|
292
|
+
split: The data split ("train" or "valid").
|
|
293
|
+
total_loss: The total loss for the epoch.
|
|
294
|
+
sub_losses: A dictionary of sub-losses for the epoch.
|
|
295
|
+
"""
|
|
296
|
+
# dataset_len = len(
|
|
297
|
+
# self._trainloader.dataset if split == "train" else self._validloader.dataset
|
|
298
|
+
# )
|
|
299
|
+
self._result.losses.add(epoch=epoch, split=split, data=total_loss)
|
|
300
|
+
self._result.sub_losses.add(
|
|
301
|
+
epoch=epoch,
|
|
302
|
+
split=split,
|
|
303
|
+
data={k: v if "_factor" not in k else v for k, v in sub_losses.items()},
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
self._fabric.print(
|
|
307
|
+
f"Epoch {epoch + 1} - {split.capitalize()} Loss: {total_loss:.4f}"
|
|
308
|
+
)
|
|
309
|
+
self._fabric.print(
|
|
310
|
+
f"Sub-losses: {', '.join([f'{k}: {v:.4f}' for k, v in sub_losses.items()])}"
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
def _store_checkpoint(self, epoch: int) -> None:
|
|
314
|
+
"""Stores model checkpoints and training dynamics to result object.
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
epoch: The current epoch number.
|
|
318
|
+
"""
|
|
319
|
+
self._result.model_checkpoints.add(epoch=epoch, data=self._model.state_dict())
|
|
320
|
+
if self._config.save_memory:
|
|
321
|
+
return
|
|
322
|
+
self._dynamics_to_result(epoch, "train")
|
|
323
|
+
if self._validset:
|
|
324
|
+
self._dynamics_to_result(epoch, "valid")
|
|
325
|
+
|
|
326
|
+
def _capture_dynamics(
|
|
327
|
+
self,
|
|
328
|
+
model_output: Union[ModelOutput, Any],
|
|
329
|
+
split: str,
|
|
330
|
+
indices: torch.Tensor,
|
|
331
|
+
sample_ids: Any,
|
|
332
|
+
**kwargs,
|
|
333
|
+
) -> None:
|
|
334
|
+
"""Writes model dynamics (latent space, reconstructions, etc.) to buffers.
|
|
335
|
+
|
|
336
|
+
Args:
|
|
337
|
+
model_output: The output from the model containing dynamics information.
|
|
338
|
+
split: The data split ("train", "valid", or "test").
|
|
339
|
+
indices: The indices of the samples in the current batch.
|
|
340
|
+
sample_ids: The sample IDs corresponding to the current batch.
|
|
341
|
+
**kwargs: Additional arguments (not used here).
|
|
342
|
+
|
|
343
|
+
"""
|
|
344
|
+
indices_np = (
|
|
345
|
+
indices.cpu().numpy()
|
|
346
|
+
if isinstance(indices, torch.Tensor)
|
|
347
|
+
else np.array(indices)
|
|
348
|
+
)
|
|
349
|
+
if isinstance(sample_ids, torch.Tensor):
|
|
350
|
+
sample_ids = sample_ids.cpu().numpy()
|
|
351
|
+
self._sample_ids_buffer[split][indices_np] = np.array(sample_ids)
|
|
352
|
+
if self._config.save_memory and split != "test":
|
|
353
|
+
return
|
|
354
|
+
indices_np = (
|
|
355
|
+
indices.cpu().numpy()
|
|
356
|
+
if isinstance(indices, torch.Tensor)
|
|
357
|
+
else np.array(indices)
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
self._sample_ids_buffer[split][indices_np] = np.array(sample_ids)
|
|
361
|
+
|
|
362
|
+
if self._config.save_vram: # Move to CPU to avoid GPU OOM, but will be slower
|
|
363
|
+
self._latentspace_buffer[split][
|
|
364
|
+
indices_np
|
|
365
|
+
] = model_output.latentspace.cpu().detach()
|
|
366
|
+
self._reconstruction_buffer[split][
|
|
367
|
+
indices_np
|
|
368
|
+
] = model_output.reconstruction.cpu().detach()
|
|
369
|
+
if model_output.latent_logvar is not None:
|
|
370
|
+
self._sigma_buffer[split][
|
|
371
|
+
indices_np
|
|
372
|
+
] = model_output.latent_logvar.cpu().detach()
|
|
373
|
+
if model_output.latent_mean is not None:
|
|
374
|
+
self._mu_buffer[split][
|
|
375
|
+
indices_np
|
|
376
|
+
] = model_output.latent_mean.cpu().detach()
|
|
377
|
+
else:
|
|
378
|
+
self._latentspace_buffer[split][
|
|
379
|
+
indices_np
|
|
380
|
+
] = model_output.latentspace.detach()
|
|
381
|
+
self._reconstruction_buffer[split][
|
|
382
|
+
indices_np
|
|
383
|
+
] = model_output.reconstruction.detach()
|
|
384
|
+
if model_output.latent_logvar is not None:
|
|
385
|
+
self._sigma_buffer[split][
|
|
386
|
+
indices_np
|
|
387
|
+
] = model_output.latent_logvar.detach()
|
|
388
|
+
if model_output.latent_mean is not None:
|
|
389
|
+
self._mu_buffer[split][indices_np] = model_output.latent_mean.detach()
|
|
390
|
+
|
|
391
|
+
def _dynamics_to_result(self, epoch: int, split: str) -> None:
|
|
392
|
+
"""Transfers buffered dynamics to the Result object.
|
|
393
|
+
|
|
394
|
+
Args:
|
|
395
|
+
epoch: The current epoch number.
|
|
396
|
+
split: The data split ("train", "valid", or "test").
|
|
397
|
+
"""
|
|
398
|
+
|
|
399
|
+
def maybe_add(buffer, target):
|
|
400
|
+
if buffer[split] is not None and buffer[split].sum() != 0:
|
|
401
|
+
target.add(
|
|
402
|
+
epoch=epoch, split=split, data=buffer[split].cpu().detach().numpy()
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
self._result.latentspaces.add(
|
|
406
|
+
epoch=epoch,
|
|
407
|
+
split=split,
|
|
408
|
+
data=self._latentspace_buffer[split].cpu().detach().numpy(),
|
|
409
|
+
)
|
|
410
|
+
self._result.reconstructions.add(
|
|
411
|
+
epoch=epoch,
|
|
412
|
+
split=split,
|
|
413
|
+
data=self._reconstruction_buffer[split].cpu().detach().numpy(),
|
|
414
|
+
)
|
|
415
|
+
self._result.sample_ids.add(
|
|
416
|
+
epoch=epoch, split=split, data=self._sample_ids_buffer[split]
|
|
417
|
+
)
|
|
418
|
+
maybe_add(self._mu_buffer, self._result.mus)
|
|
419
|
+
maybe_add(self._sigma_buffer, self._result.sigmas)
|
|
420
|
+
|
|
421
|
+
def predict(
|
|
422
|
+
self, data: BaseDataset, model: Optional[torch.nn.Module] = None, **kwargs
|
|
423
|
+
) -> Result:
|
|
424
|
+
"""Decided to add predict method to the trainer class.
|
|
425
|
+
|
|
426
|
+
This violates SRP, but the trainer class has a lot of attributes and methods
|
|
427
|
+
that are needed for prediction. So this way we don't need to write so much duplicate code
|
|
428
|
+
|
|
429
|
+
Args:
|
|
430
|
+
data: BaseDataset unseen data to run inference on
|
|
431
|
+
model: torch.nn.Module model to run inference with
|
|
432
|
+
**kwargs: Additional arguments (not used here).
|
|
433
|
+
|
|
434
|
+
Returns:
|
|
435
|
+
self._result: Result object containing the inference results
|
|
436
|
+
|
|
437
|
+
"""
|
|
438
|
+
if model is None:
|
|
439
|
+
model: torch.nn.Module = self._model
|
|
440
|
+
import warnings
|
|
441
|
+
|
|
442
|
+
warnings.warn(
|
|
443
|
+
"No model provided for prediction, using the trained model from the trainer."
|
|
444
|
+
)
|
|
445
|
+
model.eval()
|
|
446
|
+
inference_loader = DataLoader(
|
|
447
|
+
data,
|
|
448
|
+
batch_size=self._config.batch_size,
|
|
449
|
+
shuffle=False,
|
|
450
|
+
)
|
|
451
|
+
self.n_test = len(data)
|
|
452
|
+
self._init_buffers(input_data=data)
|
|
453
|
+
inference_loader = self._fabric.setup_dataloaders(inference_loader) # type: ignore
|
|
454
|
+
with self._fabric.autocast(), torch.inference_mode():
|
|
455
|
+
processed_samples = 0
|
|
456
|
+
for idx, data, sample_ids in inference_loader:
|
|
457
|
+
model_output = model(data)
|
|
458
|
+
processed_samples += len(data)
|
|
459
|
+
print(
|
|
460
|
+
f"Processed {processed_samples} / {self.n_test} samples", end="\r"
|
|
461
|
+
)
|
|
462
|
+
self._capture_dynamics(
|
|
463
|
+
model_output=model_output,
|
|
464
|
+
split="test",
|
|
465
|
+
sample_ids=sample_ids,
|
|
466
|
+
indices=idx,
|
|
467
|
+
)
|
|
468
|
+
self._dynamics_to_result(epoch=-1, split="test")
|
|
469
|
+
|
|
470
|
+
return self._result
|
|
471
|
+
|
|
472
|
+
def decode(self, x: torch.Tensor):
|
|
473
|
+
"""Decodes the input tensor x using the trained model.
|
|
474
|
+
|
|
475
|
+
Args:
|
|
476
|
+
x: Input tensor to be decoded.
|
|
477
|
+
Returns:
|
|
478
|
+
Decoded tensor.
|
|
479
|
+
"""
|
|
480
|
+
with self._fabric.autocast(), torch.no_grad():
|
|
481
|
+
x = self._fabric.to_device(obj=x)
|
|
482
|
+
if not isinstance(x, torch.Tensor):
|
|
483
|
+
raise TypeError(
|
|
484
|
+
f"Expected input to be a torch.Tensor, got {type(x)} instead."
|
|
485
|
+
)
|
|
486
|
+
return self._model.decode(x=x)
|
|
487
|
+
|
|
488
|
+
def get_model(self) -> torch.nn.Module:
|
|
489
|
+
"""Getter method for the trained model.
|
|
490
|
+
|
|
491
|
+
Returns:
|
|
492
|
+
|
|
493
|
+
The trained model as a torch.nn.Module.
|
|
494
|
+
"""
|
|
495
|
+
return self._model
|
|
496
|
+
|
|
497
|
+
def purge(self) -> None:
|
|
498
|
+
"""Cleans up any resources used during training, such as cached data or large attributes."""
|
|
499
|
+
|
|
500
|
+
attrs_to_delete = [
|
|
501
|
+
"_trainloader",
|
|
502
|
+
"_validloader",
|
|
503
|
+
"_model",
|
|
504
|
+
"_optimizer",
|
|
505
|
+
"_latentspace_buffer",
|
|
506
|
+
"_reconstruction_buffer",
|
|
507
|
+
"_mu_buffer",
|
|
508
|
+
"_sigma_buffer",
|
|
509
|
+
"_sample_ids_buffer",
|
|
510
|
+
"_trainset",
|
|
511
|
+
"_loss_fn",
|
|
512
|
+
"_validset",
|
|
513
|
+
]
|
|
514
|
+
|
|
515
|
+
for attr in attrs_to_delete:
|
|
516
|
+
if hasattr(self, attr):
|
|
517
|
+
value = getattr(self, attr)
|
|
518
|
+
# Optional: ensure non-None before deletion
|
|
519
|
+
if value is not None:
|
|
520
|
+
delattr(self, attr)
|
|
521
|
+
|
|
522
|
+
if torch.cuda.is_available():
|
|
523
|
+
torch.cuda.empty_cache()
|
|
524
|
+
|
|
525
|
+
gc.collect()
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
from typing import Optional, Type, Union, Tuple, Dict, Any, List
|
|
3
|
+
from autoencodix.trainers._general_trainer import GeneralTrainer
|
|
4
|
+
from autoencodix.base._base_dataset import BaseDataset
|
|
5
|
+
from autoencodix.base._base_loss import BaseLoss
|
|
6
|
+
from autoencodix.base._base_autoencoder import BaseAutoencoder
|
|
7
|
+
from autoencodix.utils._result import Result
|
|
8
|
+
from autoencodix.configs.default_config import DefaultConfig
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class MaskixTrainer(GeneralTrainer):
|
|
12
|
+
"""Specialized trainer for Maskix (masked-based) autoencoders.
|
|
13
|
+
|
|
14
|
+
TODO
|
|
15
|
+
Attributes:
|
|
16
|
+
Inherits all attributes from GeneralTrainer.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(
|
|
20
|
+
self,
|
|
21
|
+
trainset: Optional[BaseDataset],
|
|
22
|
+
validset: Optional[BaseDataset],
|
|
23
|
+
result: Result,
|
|
24
|
+
config: DefaultConfig,
|
|
25
|
+
model_type: Type[BaseAutoencoder],
|
|
26
|
+
loss_type: Type[BaseLoss],
|
|
27
|
+
ontologies: Optional[Union[Tuple, List]] = None,
|
|
28
|
+
masking_fn: Optional[Any] = None,
|
|
29
|
+
masking_fn_kwargs: Optional[Dict[str, Any]] = {},
|
|
30
|
+
**kwargs,
|
|
31
|
+
):
|
|
32
|
+
"""Initializes the OntixTrainer with the given datasets, model, and configuration.
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
trainset: The dataset used for training.
|
|
37
|
+
validset: The dataset used for validation, if provided.
|
|
38
|
+
result: An object to store and manage training results.
|
|
39
|
+
config: Configuration object containing training hyperparameters and settings.
|
|
40
|
+
model_type: The autoencoder model class to be trained.
|
|
41
|
+
loss_type: The loss function class specific to the model.
|
|
42
|
+
"""
|
|
43
|
+
super().__init__(
|
|
44
|
+
trainset=trainset,
|
|
45
|
+
validset=validset,
|
|
46
|
+
result=result,
|
|
47
|
+
config=config,
|
|
48
|
+
model_type=model_type,
|
|
49
|
+
loss_type=loss_type,
|
|
50
|
+
ontologies=ontologies,
|
|
51
|
+
)
|
|
52
|
+
mask_probas_list: List[float] = [
|
|
53
|
+
self._config.maskix_swap_prob
|
|
54
|
+
] * self._model.input_dim
|
|
55
|
+
self._mask_probas = torch.tensor(mask_probas_list).to(self._model.device)
|
|
56
|
+
self.masking_fn = masking_fn
|
|
57
|
+
self.masking_fn_kwargs = masking_fn_kwargs
|
|
58
|
+
if self.masking_fn is not None:
|
|
59
|
+
self._validate_masking_fn()
|
|
60
|
+
|
|
61
|
+
def _validate_masking_fn(self):
|
|
62
|
+
"""Test that user provided masking function accepts a torch.Tensor as input and returns a torch.Tensor."""
|
|
63
|
+
import inspect
|
|
64
|
+
|
|
65
|
+
sig = inspect.signature(self.masking_fn)
|
|
66
|
+
if len(sig.parameters) < 1:
|
|
67
|
+
raise ValueError(
|
|
68
|
+
"The provided masking function must accept at least one argument (the input tensor)."
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
test_input = torch.randn(100, 30)
|
|
72
|
+
test_output = self.masking_fn(test_input, **self.masking_fn_kwargs)
|
|
73
|
+
if not isinstance(test_output, torch.Tensor):
|
|
74
|
+
raise ValueError(
|
|
75
|
+
"The provided masking function must return a torch.Tensor as output."
|
|
76
|
+
)
|
|
77
|
+
if test_output.shape != test_input.shape:
|
|
78
|
+
raise ValueError(
|
|
79
|
+
"The output shape of the masking function must match the input shape."
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
def maskix_hook(self, X: torch.Tensor) -> torch.Tensor:
|
|
83
|
+
"""Applies the Maskix corruption mechanism to the input data X.
|
|
84
|
+
|
|
85
|
+
For each feature in X, with a probability defined in self._mask_probas,
|
|
86
|
+
the feature value is replaced by a value from another randomly selected sample.
|
|
87
|
+
This creates a corrupted version of the input data.
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
X: Input data tensor of shape (batch_size, num_features).
|
|
91
|
+
Returns:
|
|
92
|
+
A corrupted version of the input data tensor.
|
|
93
|
+
"""
|
|
94
|
+
if self.masking_fn is None:
|
|
95
|
+
return self._maskix_hook(X)
|
|
96
|
+
return self.masking_fn(X, **self.masking_fn_kwargs)
|
|
97
|
+
|
|
98
|
+
def _maskix_hook(
|
|
99
|
+
self, X: torch.Tensor
|
|
100
|
+
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
|
101
|
+
# expand probablities for bernoulli sampling to match input shape
|
|
102
|
+
probs = self._mask_probas.expand(X.shape)
|
|
103
|
+
|
|
104
|
+
# Create the Boolean Mask (1 = Swap, 0 = Keep)
|
|
105
|
+
should_swap = torch.bernoulli(probs).bool()
|
|
106
|
+
|
|
107
|
+
# COLUMN-WISE SHUFFLING
|
|
108
|
+
# We generate a random float matrix and argsort it along dim=0.
|
|
109
|
+
# This gives us independent random indices for every column.
|
|
110
|
+
rand_indices = torch.rand(X.shape, device=X.device).argsort(dim=0)
|
|
111
|
+
|
|
112
|
+
# Use gather to reorder X based on these random indices
|
|
113
|
+
shuffled_X = torch.gather(X, 0, rand_indices)
|
|
114
|
+
corrupted_X = torch.where(should_swap, shuffled_X, X)
|
|
115
|
+
|
|
116
|
+
return corrupted_X
|