autoencodix 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of autoencodix might be problematic. Click here for more details.

Files changed (91) hide show
  1. autoencodix/__init__.py +27 -0
  2. autoencodix/base/.ipynb_checkpoints/_base_pipeline-checkpoint.py +1274 -0
  3. autoencodix/base/.ipynb_checkpoints/_base_trainer-checkpoint.py +247 -0
  4. autoencodix/base/__init__.py +21 -0
  5. autoencodix/base/_base_autoencoder.py +138 -0
  6. autoencodix/base/_base_dataset.py +103 -0
  7. autoencodix/base/_base_evaluator.py +34 -0
  8. autoencodix/base/_base_loss.py +311 -0
  9. autoencodix/base/_base_pipeline.py +1355 -0
  10. autoencodix/base/_base_predictor.py +6 -0
  11. autoencodix/base/_base_preprocessor.py +1033 -0
  12. autoencodix/base/_base_trainer.py +259 -0
  13. autoencodix/base/_base_visualizer.py +578 -0
  14. autoencodix/base/sampleconfig.yaml +132 -0
  15. autoencodix/configs/__init__.py +21 -0
  16. autoencodix/configs/default_config.py +652 -0
  17. autoencodix/configs/disentanglix_config.py +15 -0
  18. autoencodix/configs/maskix_config.py +39 -0
  19. autoencodix/configs/ontix_config.py +35 -0
  20. autoencodix/configs/stackix_config.py +33 -0
  21. autoencodix/configs/vanillix_config.py +18 -0
  22. autoencodix/configs/varix_config.py +15 -0
  23. autoencodix/configs/xmodalix_config.py +43 -0
  24. autoencodix/disentanglix.py +72 -0
  25. autoencodix/evaluate/.ipynb_checkpoints/evaluate-checkpoint.py +532 -0
  26. autoencodix/evaluate/__init__.py +4 -0
  27. autoencodix/evaluate/_general_evaluator.py +730 -0
  28. autoencodix/evaluate/_xmodalix_evaluator.py +376 -0
  29. autoencodix/imagix.py +76 -0
  30. autoencodix/losses/disentanglix_loss.py +215 -0
  31. autoencodix/losses/maskix_loss.py +94 -0
  32. autoencodix/losses/vanillix_loss.py +28 -0
  33. autoencodix/losses/varix_loss.py +102 -0
  34. autoencodix/losses/xmodal_loss.py +304 -0
  35. autoencodix/maskix.py +108 -0
  36. autoencodix/modeling/.ipynb_checkpoints/__init__-checkpoint.py +16 -0
  37. autoencodix/modeling/.ipynb_checkpoints/_captum_forward-checkpoint.py +16 -0
  38. autoencodix/modeling/__init__.py +20 -0
  39. autoencodix/modeling/_captum_forward.py +20 -0
  40. autoencodix/modeling/_classifier.py +45 -0
  41. autoencodix/modeling/_imagevae_architecture.py +296 -0
  42. autoencodix/modeling/_imgfast_architecture.py +288 -0
  43. autoencodix/modeling/_layer_factory.py +71 -0
  44. autoencodix/modeling/_maskix_architecture.py +190 -0
  45. autoencodix/modeling/_ontix_architecture.py +305 -0
  46. autoencodix/modeling/_vanillix_architecture.py +136 -0
  47. autoencodix/modeling/_varix_architecture.py +192 -0
  48. autoencodix/notebooks/__init__.py +0 -0
  49. autoencodix/ontix.py +136 -0
  50. autoencodix/py.typed +0 -0
  51. autoencodix/stackix.py +122 -0
  52. autoencodix/trainers/__init__.py +6 -0
  53. autoencodix/trainers/_general_trainer.py +525 -0
  54. autoencodix/trainers/_maskix_trainer.py +116 -0
  55. autoencodix/trainers/_ontix_trainer.py +77 -0
  56. autoencodix/trainers/_stackix_orchestrator.py +522 -0
  57. autoencodix/trainers/_stackix_trainer.py +209 -0
  58. autoencodix/trainers/_xmodal_trainer.py +1006 -0
  59. autoencodix/utils/.ipynb_checkpoints/_explainer-checkpoint.py +118 -0
  60. autoencodix/utils/.ipynb_checkpoints/_model_output-checkpoint.py +30 -0
  61. autoencodix/utils/.ipynb_checkpoints/_utils-checkpoint.py +626 -0
  62. autoencodix/utils/.ipynb_checkpoints/adata_converter-checkpoint.py +99 -0
  63. autoencodix/utils/.ipynb_checkpoints/feature_importance-checkpoint.py +823 -0
  64. autoencodix/utils/__init__.py +22 -0
  65. autoencodix/utils/_annealer.py +193 -0
  66. autoencodix/utils/_bulkreader.py +232 -0
  67. autoencodix/utils/_explainer.py +246 -0
  68. autoencodix/utils/_imgreader.py +404 -0
  69. autoencodix/utils/_internals.py +23 -0
  70. autoencodix/utils/_llm_explainer.py +318 -0
  71. autoencodix/utils/_losses.py +20 -0
  72. autoencodix/utils/_model_output.py +30 -0
  73. autoencodix/utils/_result.py +393 -0
  74. autoencodix/utils/_screader.py +76 -0
  75. autoencodix/utils/_traindynamics.py +187 -0
  76. autoencodix/utils/_utils.py +690 -0
  77. autoencodix/utils/adata_converter.py +111 -0
  78. autoencodix/utils/example_data.py +398 -0
  79. autoencodix/utils/feature_importance.py +829 -0
  80. autoencodix/utils/prompts.py +21 -0
  81. autoencodix/vanillix.py +141 -0
  82. autoencodix/varix.py +80 -0
  83. autoencodix/visualize/__init__.py +5 -0
  84. autoencodix/visualize/_general_visualizer.py +913 -0
  85. autoencodix/visualize/_imagix_visualizer.py +111 -0
  86. autoencodix/visualize/_xmodal_visualizer.py +841 -0
  87. autoencodix/visualize/visualize.py +1214 -0
  88. autoencodix/xmodalix.py +318 -0
  89. autoencodix-0.2.0.dist-info/METADATA +143 -0
  90. autoencodix-0.2.0.dist-info/RECORD +91 -0
  91. autoencodix-0.2.0.dist-info/WHEEL +4 -0
@@ -0,0 +1,20 @@
1
+ from ._layer_factory import LayerFactory
2
+ from ._vanillix_architecture import VanillixArchitecture
3
+ from ._varix_architecture import VarixArchitecture
4
+ from ._ontix_architecture import OntixArchitecture
5
+ from ._imagevae_architecture import ImageVAEArchitecture
6
+ from ._classifier import Classifier
7
+ from ._maskix_architecture import MaskixArchitectureVanilla
8
+ from ._imgfast_architecture import ImageVAEFastArchitecture
9
+
10
+ # test done
11
+ __all__ = [
12
+ "LayerFactory",
13
+ "VanillixArchitecture",
14
+ "VarixArchitecture",
15
+ "OntixArchitecture",
16
+ "ImageVAEArchitecture",
17
+ "Classifier",
18
+ "MaskixArchitectureVanilla",
19
+ "ImageVAEFastArchitecture",
20
+ ]
@@ -0,0 +1,20 @@
1
+ import torch
2
+ import torch.nn as nn
3
+ from autoencodix.base._base_autoencoder import BaseAutoencoder
4
+ from autoencodix.utils._model_output import ModelOutput
5
+
6
+
7
+ class CaptumForward(nn.Module):
8
+ def __init__(self, model: BaseAutoencoder, dim: int):
9
+ super(CaptumForward, self).__init__() # <-- REQUIRED
10
+ self.model = model # (Registered as a submodule)
11
+ self.dim = dim
12
+ self.device = next(
13
+ model.parameters()
14
+ ).device # Get the device of the model parameters
15
+
16
+ def forward(self, x: torch.Tensor):
17
+ mp: ModelOutput = self.model(x=x.to(self.device))
18
+ latent = mp.latentspace
19
+ output = latent[:, self.dim]
20
+ return output.unsqueeze(1).to("cpu")
@@ -0,0 +1,45 @@
1
+ import torch
2
+ import torch.nn as nn
3
+
4
+
5
+ class Classifier(nn.Module):
6
+ """Multi-class classifier for adversarial training in n-modal latent space alignment.
7
+
8
+
9
+ Attributes:
10
+ input_dim: Dimension of the input features.
11
+ n_modalities: Number of modalities (classes) to classify.
12
+ n_hidden: Number of hidden units in the classifier network.
13
+
14
+ """
15
+
16
+ def __init__(self, input_dim: int, n_modalities: int, n_hidden: int = 64) -> None:
17
+ super().__init__()
18
+ self.input_dim = input_dim
19
+ self.n_modalities = n_modalities
20
+ self.classifier = nn.Sequential(
21
+ nn.Linear(input_dim, n_hidden),
22
+ nn.ReLU(inplace=False),
23
+ nn.Dropout(0.1),
24
+ nn.Linear(n_hidden, n_hidden // 2),
25
+ nn.ReLU(inplace=False),
26
+ nn.Dropout(0.1),
27
+ nn.Linear(n_hidden // 2, n_modalities),
28
+ )
29
+ self.apply(self._init_weights)
30
+
31
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
32
+ """Forward pass through the classifier.
33
+
34
+ Args:
35
+ x: Input tensor of shape (batch_size, input_dim).
36
+ Returns:
37
+ Output tensor of shape (batch_size, n_modalities) representing class scores.
38
+ """
39
+ return self.classifier(x)
40
+
41
+ def _init_weights(self, m: nn.Module) -> None:
42
+ if isinstance(m, nn.Linear):
43
+ nn.init.xavier_uniform_(m.weight)
44
+ if m.bias is not None:
45
+ m.bias.data.fill_(0.01)
@@ -0,0 +1,296 @@
1
+ import torch
2
+ import torch.nn as nn
3
+ from typing import Tuple, Optional, Union, Dict
4
+ from autoencodix.configs.default_config import DefaultConfig
5
+ from autoencodix.utils._model_output import ModelOutput
6
+ from autoencodix.base._base_autoencoder import BaseAutoencoder
7
+
8
+
9
+ class ImageVAEArchitecture(BaseAutoencoder):
10
+ """This class defines a VAE, based on a CNN for images
11
+
12
+ It takes as input an image and of shape (C,W,H) and reconstructs it.
13
+ We ensure to have a latent space of shape <batchsize,1,LatentDim> and img_in.shape = img_out.shape
14
+ We have a fixed kernel_size=4, padding=1 and stride=2 (given from https://github.com/uhlerlab/cross-modal-auto_encoders/tree/master)
15
+
16
+ So we need to calculate how the image dimension changes after each Convolution (we assume W=H)
17
+ Applying the formular:
18
+ W_out = (((W - kernel_size + 2padding)/stride) + 1)
19
+ We get:
20
+ W_out = (((W-4+2*1)/2)+1) =
21
+ = (W-2/2)+1 =
22
+ = (2(0.5W-1)/2) +1 # factor 2 out
23
+ = 0.5W - 1 + 1
24
+ W_out = 0.5W
25
+ So in this configuration the output shape halfs after every convolutional step (assuming W=H)
26
+
27
+
28
+ Attributes:
29
+ input_dim: (C,W,H) the input image shape
30
+ config: Configuration object containing model architecture parameters
31
+ _encoder: Encoder network of the autoencoder
32
+ _decoder: Decoder network of the autoencoder
33
+ latent_dim: Dimension of the latent space
34
+ nc: number of channels in the input image
35
+ h: height of the input image
36
+ w: width of the input image
37
+ img_shape: (C,W,H) the input image shape
38
+ hidden_dim: number of filters in the first convolutional layer
39
+ """
40
+
41
+ def __init__(
42
+ self,
43
+ input_dim: Tuple[int, int, int], # (C,W,H) the input image shape
44
+ config: Optional[DefaultConfig],
45
+ ontologies: Optional[Union[Tuple, Dict]] = None,
46
+ feature_order: Optional[Union[Tuple, Dict]] = None,
47
+ # the input_dim is the number of channels in the image, e.g. 3
48
+ ):
49
+ """Initialize the ImageVAEArchitecture with the given configuration.
50
+
51
+ Args:
52
+ input_dim: (C,W,H) the input image shape
53
+ config: Configuration object containing model parameters.
54
+ hidden_dim: number of filters in the first convolutional layer
55
+ """
56
+ if config is None:
57
+ config = DefaultConfig()
58
+ self._config: DefaultConfig = config
59
+ super().__init__(config=config, input_dim=input_dim)
60
+ self.input_dim: int = input_dim
61
+ self.latent_dim: int = self._config.latent_dim
62
+ self.nc, self.h, self.w = input_dim
63
+ self.img_shape: Tuple[int, int, int] = input_dim
64
+ self.hidden_dim: int = self._config.hidden_dim
65
+ self._build_network()
66
+ self.apply(self._init_weights)
67
+
68
+ def _build_network(self):
69
+ """Construct the encoder and decoder networks."""
70
+ self._encoder = nn.Sequential(
71
+ nn.Conv2d(
72
+ in_channels=self.nc,
73
+ out_channels=self.hidden_dim,
74
+ kernel_size=4,
75
+ stride=2,
76
+ padding=1,
77
+ bias=False,
78
+ ),
79
+ nn.LeakyReLU(0.2, inplace=False),
80
+ nn.Conv2d(
81
+ in_channels=self.hidden_dim,
82
+ out_channels=self.hidden_dim * 2,
83
+ kernel_size=4,
84
+ stride=2,
85
+ padding=1,
86
+ bias=False,
87
+ ),
88
+ nn.BatchNorm2d(self.hidden_dim * 2),
89
+ nn.LeakyReLU(0.2, inplace=False),
90
+ nn.Conv2d(
91
+ in_channels=self.hidden_dim * 2,
92
+ out_channels=self.hidden_dim * 4,
93
+ kernel_size=4,
94
+ stride=2,
95
+ padding=1,
96
+ bias=False,
97
+ ),
98
+ nn.BatchNorm2d(self.hidden_dim * 4),
99
+ nn.LeakyReLU(0.2, inplace=False),
100
+ nn.Conv2d(
101
+ in_channels=self.hidden_dim * 4,
102
+ out_channels=self.hidden_dim * 8,
103
+ kernel_size=4,
104
+ stride=2,
105
+ padding=1,
106
+ bias=False,
107
+ ),
108
+ nn.BatchNorm2d(self.hidden_dim * 8),
109
+ nn.LeakyReLU(0.2, inplace=False),
110
+ nn.Conv2d(
111
+ in_channels=self.hidden_dim * 8,
112
+ out_channels=self.hidden_dim * 8,
113
+ kernel_size=4,
114
+ stride=2,
115
+ padding=1,
116
+ bias=False,
117
+ ),
118
+ nn.BatchNorm2d(self.hidden_dim * 8),
119
+ nn.LeakyReLU(0.2, inplace=False),
120
+ )
121
+
122
+ # to Calculate the image shape after the _encoder, we need to know the number of layers
123
+ # because the shape halfs after every Conv2D layer
124
+ self.num__encoder_layers = sum(
125
+ 1 for _ in self._encoder.children() if isinstance(_, nn.Conv2d)
126
+ )
127
+ # So the output shape after all layers is in_shape / 2**N_layers
128
+ # We showed above in the DocString why the shape halfs
129
+ self.spatial_dim = self.h // (2**self.num__encoder_layers)
130
+ # In the Linear mu and logvar layer we need to flatten the 3D output to a 2D matrix
131
+ # Therefore we need to multiply the size of every out diemension of the input layer to the Linear layers
132
+ # This is hidden_dim * 8 (the number of filter/channel layer) * spatial dim (the widht of the image) * spatial diem (the height of the image)
133
+ # assuimg width = height
134
+ # The original paper had a fixed spatial dimension of 2, which only worked for images with 64x64 shape
135
+ self.mu = nn.Linear(
136
+ self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim, self.latent_dim
137
+ )
138
+ self.logvar = nn.Linear(
139
+ self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim, self.latent_dim
140
+ )
141
+
142
+ # the same logic goes for the first _decoder layer, which takes the latent_dim as inshape
143
+ # which is the outshape of the previous mu/logvar layer
144
+ # and the shape of the first ConvTranspose2D layer is the last outpus shape of the _encoder layer
145
+ # This the same multiplication as above
146
+ self.d1 = nn.Sequential(
147
+ nn.Linear(
148
+ self.latent_dim,
149
+ self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim,
150
+ ),
151
+ nn.ReLU(inplace=False),
152
+ )
153
+ self._decoder = nn.Sequential(
154
+ nn.ConvTranspose2d(
155
+ in_channels=self.hidden_dim * 8,
156
+ out_channels=self.hidden_dim * 8,
157
+ kernel_size=4,
158
+ stride=2,
159
+ padding=1,
160
+ bias=False,
161
+ ),
162
+ nn.BatchNorm2d(self.hidden_dim * 8),
163
+ nn.LeakyReLU(0.2, inplace=False),
164
+ nn.ConvTranspose2d(
165
+ in_channels=self.hidden_dim * 8,
166
+ out_channels=self.hidden_dim * 4,
167
+ kernel_size=4,
168
+ stride=2,
169
+ padding=1,
170
+ bias=False,
171
+ ),
172
+ nn.BatchNorm2d(self.hidden_dim * 4),
173
+ nn.LeakyReLU(0.2, inplace=False),
174
+ nn.ConvTranspose2d(
175
+ in_channels=self.hidden_dim * 4,
176
+ out_channels=self.hidden_dim * 2,
177
+ kernel_size=4,
178
+ stride=2,
179
+ padding=1,
180
+ bias=False,
181
+ ),
182
+ nn.BatchNorm2d(self.hidden_dim * 2),
183
+ nn.LeakyReLU(0.2, inplace=False),
184
+ nn.ConvTranspose2d(
185
+ in_channels=self.hidden_dim * 2,
186
+ out_channels=self.hidden_dim,
187
+ kernel_size=4,
188
+ stride=2,
189
+ padding=1,
190
+ bias=False,
191
+ ),
192
+ nn.BatchNorm2d(self.hidden_dim),
193
+ nn.LeakyReLU(0.2, inplace=False),
194
+ nn.ConvTranspose2d(
195
+ in_channels=self.hidden_dim,
196
+ out_channels=self.nc,
197
+ kernel_size=4,
198
+ stride=2,
199
+ padding=1,
200
+ bias=False,
201
+ ),
202
+ )
203
+
204
+ def _get_spatial_dim(self) -> int:
205
+ return self.spatial_dim
206
+
207
+ def encode(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
208
+ """Encodes the input tensor x.
209
+
210
+ Args:
211
+ x: Input tensor
212
+ Returns:
213
+ The encoded latent space representation, or mu and logvar for VAEs.
214
+
215
+ """
216
+ h = self._encoder(x)
217
+ # this makes sure we get the <batchsize, 1, latent_dim> shape for our latent space in the next step
218
+ # because we put all dimensionaltiy in the second dimension of the output shape.
219
+ # By covering all dimensionality here, we are sure that the rest is
220
+ h = h.view(-1, self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim)
221
+ logvar = self.logvar(h)
222
+ mu = self.mu(h)
223
+ # prevent mu and logvar from being too close to zero, this increased
224
+ # numerical stability
225
+ logvar = torch.clamp(logvar, 0.1, 20)
226
+ # replace mu when mu < 0.00000001 with 0.1
227
+ mu = torch.where(mu < 0.000001, torch.zeros_like(mu), mu)
228
+ return mu, logvar
229
+
230
+ def reparameterize(self, mu: torch.Tensor, logvar: torch.Tensor) -> torch.Tensor:
231
+ """Reparameterization trick for VAE.
232
+
233
+ Args:
234
+ mu: mean of the latent distribution
235
+ logvar: log-variance of the latent distribution
236
+ Returns:
237
+ z: sampled latent vector
238
+ """
239
+ std = torch.exp(0.5 * logvar)
240
+ eps = torch.randn_like(std)
241
+ return mu + eps * std
242
+
243
+ def get_latent_space(self, x: torch.Tensor) -> torch.Tensor:
244
+ """Returns the latent space representation of the input.
245
+
246
+ Args:
247
+ x: Input tensor
248
+ Returns:
249
+ Latent space representation
250
+
251
+ """
252
+ mu, logvar = self.encode(x)
253
+ return self.reparameterize(mu, logvar)
254
+
255
+ def decode(self, x: torch.Tensor) -> torch.Tensor:
256
+ """Decode the latent tensor x
257
+ Args:
258
+ x: Latent tensor
259
+ Returns:
260
+ Decoded tensor, reconstructed from the latent space
261
+ """
262
+ h = self.d1(x)
263
+ # here we do a similar thing as in the _encoder,
264
+ # but instead of ensuring the correct dimension for the latent space,
265
+ # we ensure the correct dimension for the first Conv2DTranspose layer
266
+ # so we make sure that the last 3 dimension are (n_filters, reduced_img_dim, reduced_img_dim)
267
+ h = h.view(-1, self.hidden_dim * 8, self.spatial_dim, self.spatial_dim)
268
+ return self._decoder(h)
269
+
270
+ def translate(self, z: torch.Tensor) -> torch.Tensor:
271
+ """Reshapes the output to get actual images
272
+
273
+ Args:
274
+ z: Latent tensor
275
+ Returns:
276
+ Reconstructed image of shape (C,W,H)
277
+ """
278
+ out = self.decode(z)
279
+ return out.view(-1, *self.img_shape)
280
+
281
+ def forward(self, x: torch.Tensor) -> ModelOutput:
282
+ """Forward pass of the model.
283
+ Args:
284
+ x: Input tensor
285
+ Returns:
286
+ ModelOutput object containing the reconstructed tensor and latent tensor
287
+ """
288
+ mu, logvar = self.encode(x)
289
+ z = self.reparameterize(mu, logvar)
290
+ return ModelOutput(
291
+ reconstruction=self.translate(z),
292
+ latentspace=z,
293
+ latent_mean=mu,
294
+ latent_logvar=logvar,
295
+ additional_info=None,
296
+ )
@@ -0,0 +1,288 @@
1
+ import torch
2
+ import torch.nn as nn
3
+ from typing import Tuple, Optional, Union, Dict
4
+ from autoencodix.configs.default_config import DefaultConfig
5
+ from autoencodix.utils._model_output import ModelOutput
6
+ from autoencodix.base._base_autoencoder import BaseAutoencoder
7
+
8
+
9
+ class ImageVAEFastArchitecture(BaseAutoencoder):
10
+ """This class defines a VAE, based on a CNN for images
11
+
12
+ It takes as input an image and of shape (C,W,H) and reconstructs it.
13
+ We ensure to have a latent space of shape <batchsize,1,LatentDim> and img_in.shape = img_out.shape
14
+ We have a fixed kernel_size=4, padding=1 and stride=2 (given from https://github.com/uhlerlab/cross-modal-auto_encoders/tree/master)
15
+
16
+ So we need to calculate how the image dimension changes after each Convolution (we assume W=H)
17
+ Applying the formular:
18
+ W_out = (((W - kernel_size + 2padding)/stride) + 1)
19
+ We get:
20
+ W_out = (((W-4+2*1)/2)+1) =
21
+ = (W-2/2)+1 =
22
+ = (2(0.5W-1)/2) +1 # factor 2 out
23
+ = 0.5W - 1 + 1
24
+ W_out = 0.5W
25
+ So in this configuration the output shape halfs after every convolutional step (assuming W=H)
26
+
27
+
28
+ Attributes:
29
+ input_dim: (C,W,H) the input image shape
30
+ config: Configuration object containing model architecture parameters
31
+ _encoder: Encoder network of the autoencoder
32
+ _decoder: Decoder network of the autoencoder
33
+ latent_dim: Dimension of the latent space
34
+ nc: number of channels in the input image
35
+ h: height of the input image
36
+ w: width of the input image
37
+ img_shape: (C,W,H) the input image shape
38
+ hidden_dim: number of filters in the first convolutional layer
39
+ """
40
+
41
+ def __init__(
42
+ self,
43
+ input_dim: Tuple[int, int, int], # (C,W,H) the input image shape
44
+ config: Optional[DefaultConfig],
45
+ ontologies: Optional[Union[Tuple, Dict]] = None,
46
+ feature_order: Optional[Union[Tuple, Dict]] = None,
47
+ # the input_dim is the number of channels in the image, e.g. 3
48
+ ):
49
+ """Initialize the ImageVAEArchitecture with the given configuration.
50
+
51
+ Args:
52
+ input_dim: (C,W,H) the input image shape
53
+ config: Configuration object containing model parameters.
54
+ hidden_dim: number of filters in the first convolutional layer
55
+ """
56
+ if config is None:
57
+ config = DefaultConfig()
58
+ self._config: DefaultConfig = config
59
+ super().__init__(config=config, input_dim=input_dim)
60
+ self.input_dim: int = input_dim
61
+ self.latent_dim: int = self._config.latent_dim
62
+ self.nc, self.h, self.w = input_dim
63
+ self.img_shape: Tuple[int, int, int] = input_dim
64
+ self.hidden_dim: int = self._config.hidden_dim
65
+ self._build_network()
66
+ self.apply(self._init_weights)
67
+
68
+ def _build_network(self):
69
+ """Construct the encoder and decoder networks."""
70
+ self._encoder = nn.Sequential(
71
+ nn.Conv2d(
72
+ in_channels=self.nc,
73
+ out_channels=self.hidden_dim,
74
+ kernel_size=4,
75
+ stride=2,
76
+ padding=1,
77
+ bias=False,
78
+ ),
79
+ nn.LeakyReLU(0.2, inplace=True),
80
+ nn.Conv2d(
81
+ in_channels=self.hidden_dim,
82
+ out_channels=self.hidden_dim * 2,
83
+ kernel_size=4,
84
+ stride=2,
85
+ padding=1,
86
+ bias=False,
87
+ ),
88
+ nn.LeakyReLU(0.2, inplace=True),
89
+ nn.Conv2d(
90
+ in_channels=self.hidden_dim * 2,
91
+ out_channels=self.hidden_dim * 4,
92
+ kernel_size=4,
93
+ stride=2,
94
+ padding=1,
95
+ bias=False,
96
+ ),
97
+ nn.LeakyReLU(0.2, inplace=True),
98
+ nn.Conv2d(
99
+ in_channels=self.hidden_dim * 4,
100
+ out_channels=self.hidden_dim * 8,
101
+ kernel_size=4,
102
+ stride=2,
103
+ padding=1,
104
+ bias=False,
105
+ ),
106
+ nn.LeakyReLU(0.2, inplace=True),
107
+ nn.Conv2d(
108
+ in_channels=self.hidden_dim * 8,
109
+ out_channels=self.hidden_dim * 8,
110
+ kernel_size=4,
111
+ stride=2,
112
+ padding=1,
113
+ bias=False,
114
+ ),
115
+ nn.LeakyReLU(0.2, inplace=True),
116
+ )
117
+
118
+ # to Calculate the image shape after the _encoder, we need to know the number of layers
119
+ # because the shape halfs after every Conv2D layer
120
+ self.num__encoder_layers = sum(
121
+ 1 for _ in self._encoder.children() if isinstance(_, nn.Conv2d)
122
+ )
123
+ # So the output shape after all layers is in_shape / 2**N_layers
124
+ # We showed above in the DocString why the shape halfs
125
+ self.spatial_dim = self.h // (2**self.num__encoder_layers)
126
+ # In the Linear mu and logvar layer we need to flatten the 3D output to a 2D matrix
127
+ # Therefore we need to multiply the size of every out diemension of the input layer to the Linear layers
128
+ # This is hidden_dim * 8 (the number of filter/channel layer) * spatial dim (the widht of the image) * spatial diem (the height of the image)
129
+ # assuimg width = height
130
+ # The original paper had a fixed spatial dimension of 2, which only worked for images with 64x64 shape
131
+ self.mu = nn.Linear(
132
+ self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim, self.latent_dim
133
+ )
134
+ self.logvar = nn.Linear(
135
+ self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim, self.latent_dim
136
+ )
137
+
138
+ # the same logic goes for the first _decoder layer, which takes the latent_dim as inshape
139
+ # which is the outshape of the previous mu/logvar layer
140
+ # and the shape of the first ConvTranspose2D layer is the last outpus shape of the _encoder layer
141
+ # This the same multiplication as above
142
+ self.d1 = nn.Sequential(
143
+ nn.Linear(
144
+ self.latent_dim,
145
+ self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim,
146
+ ),
147
+ nn.ReLU(inplace=False),
148
+ )
149
+ self._decoder = nn.Sequential(
150
+ nn.ConvTranspose2d(
151
+ in_channels=self.hidden_dim * 8,
152
+ out_channels=self.hidden_dim * 8,
153
+ kernel_size=4,
154
+ stride=2,
155
+ padding=1,
156
+ bias=False,
157
+ ),
158
+ nn.LeakyReLU(0.2, inplace=True),
159
+ nn.ConvTranspose2d(
160
+ in_channels=self.hidden_dim * 8,
161
+ out_channels=self.hidden_dim * 4,
162
+ kernel_size=4,
163
+ stride=2,
164
+ padding=1,
165
+ bias=False,
166
+ ),
167
+ nn.LeakyReLU(0.2, inplace=True),
168
+ nn.ConvTranspose2d(
169
+ in_channels=self.hidden_dim * 4,
170
+ out_channels=self.hidden_dim * 2,
171
+ kernel_size=4,
172
+ stride=2,
173
+ padding=1,
174
+ bias=False,
175
+ ),
176
+ nn.LeakyReLU(0.2, inplace=True),
177
+ nn.ConvTranspose2d(
178
+ in_channels=self.hidden_dim * 2,
179
+ out_channels=self.hidden_dim,
180
+ kernel_size=4,
181
+ stride=2,
182
+ padding=1,
183
+ bias=False,
184
+ ),
185
+ nn.LeakyReLU(0.2, inplace=True),
186
+ nn.ConvTranspose2d(
187
+ in_channels=self.hidden_dim,
188
+ out_channels=self.nc,
189
+ kernel_size=4,
190
+ stride=2,
191
+ padding=1,
192
+ bias=False,
193
+ ),
194
+ )
195
+
196
+ def _get_spatial_dim(self) -> int:
197
+ return self.spatial_dim
198
+
199
+ def encode(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
200
+ """Encodes the input tensor x.
201
+
202
+ Args:
203
+ x: Input tensor
204
+ Returns:
205
+ The encoded latent space representation, or mu and logvar for VAEs.
206
+
207
+ """
208
+ h = self._encoder(x)
209
+ # this makes sure we get the <batchsize, 1, latent_dim> shape for our latent space in the next step
210
+ # because we put all dimensionaltiy in the second dimension of the output shape.
211
+ # By covering all dimensionality here, we are sure that the rest is
212
+ h = h.view(-1, self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim)
213
+ logvar = self.logvar(h)
214
+ mu = self.mu(h)
215
+ # prevent mu and logvar from being too close to zero, this increased
216
+ # numerical stability
217
+ logvar = torch.clamp(logvar, 0.1, 20)
218
+ # replace mu when mu < 0.00000001 with 0.1
219
+ mu = torch.where(mu < 0.000001, torch.zeros_like(mu), mu)
220
+ return mu, logvar
221
+
222
+ def reparameterize(self, mu: torch.Tensor, logvar: torch.Tensor) -> torch.Tensor:
223
+ """Reparameterization trick for VAE.
224
+
225
+ Args:
226
+ mu: mean of the latent distribution
227
+ logvar: log-variance of the latent distribution
228
+ Returns:
229
+ z: sampled latent vector
230
+ """
231
+ std = torch.exp(0.5 * logvar)
232
+ eps = torch.randn_like(std)
233
+ return mu + eps * std
234
+
235
+ def get_latent_space(self, x: torch.Tensor) -> torch.Tensor:
236
+ """Returns the latent space representation of the input.
237
+
238
+ Args:
239
+ x: Input tensor
240
+ Returns:
241
+ Latent space representation
242
+
243
+ """
244
+ mu, logvar = self.encode(x)
245
+ return self.reparameterize(mu, logvar)
246
+
247
+ def decode(self, x: torch.Tensor) -> torch.Tensor:
248
+ """Decode the latent tensor x
249
+ Args:
250
+ x: Latent tensor
251
+ Returns:
252
+ Decoded tensor, reconstructed from the latent space
253
+ """
254
+ h = self.d1(x)
255
+ # here we do a similar thing as in the _encoder,
256
+ # but instead of ensuring the correct dimension for the latent space,
257
+ # we ensure the correct dimension for the first Conv2DTranspose layer
258
+ # so we make sure that the last 3 dimension are (n_filters, reduced_img_dim, reduced_img_dim)
259
+ h = h.view(-1, self.hidden_dim * 8, self.spatial_dim, self.spatial_dim)
260
+ return self._decoder(h)
261
+
262
+ def translate(self, z: torch.Tensor) -> torch.Tensor:
263
+ """Reshapes the output to get actual images
264
+
265
+ Args:
266
+ z: Latent tensor
267
+ Returns:
268
+ Reconstructed image of shape (C,W,H)
269
+ """
270
+ out = self.decode(z)
271
+ return out.view(-1, *self.img_shape)
272
+
273
+ def forward(self, x: torch.Tensor) -> ModelOutput:
274
+ """Forward pass of the model.
275
+ Args:
276
+ x: Input tensor
277
+ Returns:
278
+ ModelOutput object containing the reconstructed tensor and latent tensor
279
+ """
280
+ mu, logvar = self.encode(x)
281
+ z = self.reparameterize(mu, logvar)
282
+ return ModelOutput(
283
+ reconstruction=self.translate(z),
284
+ latentspace=z,
285
+ latent_mean=mu,
286
+ latent_logvar=logvar,
287
+ additional_info=None,
288
+ )