autoencodix 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of autoencodix might be problematic. Click here for more details.
- autoencodix/__init__.py +27 -0
- autoencodix/base/.ipynb_checkpoints/_base_pipeline-checkpoint.py +1274 -0
- autoencodix/base/.ipynb_checkpoints/_base_trainer-checkpoint.py +247 -0
- autoencodix/base/__init__.py +21 -0
- autoencodix/base/_base_autoencoder.py +138 -0
- autoencodix/base/_base_dataset.py +103 -0
- autoencodix/base/_base_evaluator.py +34 -0
- autoencodix/base/_base_loss.py +311 -0
- autoencodix/base/_base_pipeline.py +1355 -0
- autoencodix/base/_base_predictor.py +6 -0
- autoencodix/base/_base_preprocessor.py +1033 -0
- autoencodix/base/_base_trainer.py +259 -0
- autoencodix/base/_base_visualizer.py +578 -0
- autoencodix/base/sampleconfig.yaml +132 -0
- autoencodix/configs/__init__.py +21 -0
- autoencodix/configs/default_config.py +652 -0
- autoencodix/configs/disentanglix_config.py +15 -0
- autoencodix/configs/maskix_config.py +39 -0
- autoencodix/configs/ontix_config.py +35 -0
- autoencodix/configs/stackix_config.py +33 -0
- autoencodix/configs/vanillix_config.py +18 -0
- autoencodix/configs/varix_config.py +15 -0
- autoencodix/configs/xmodalix_config.py +43 -0
- autoencodix/disentanglix.py +72 -0
- autoencodix/evaluate/.ipynb_checkpoints/evaluate-checkpoint.py +532 -0
- autoencodix/evaluate/__init__.py +4 -0
- autoencodix/evaluate/_general_evaluator.py +730 -0
- autoencodix/evaluate/_xmodalix_evaluator.py +376 -0
- autoencodix/imagix.py +76 -0
- autoencodix/losses/disentanglix_loss.py +215 -0
- autoencodix/losses/maskix_loss.py +94 -0
- autoencodix/losses/vanillix_loss.py +28 -0
- autoencodix/losses/varix_loss.py +102 -0
- autoencodix/losses/xmodal_loss.py +304 -0
- autoencodix/maskix.py +108 -0
- autoencodix/modeling/.ipynb_checkpoints/__init__-checkpoint.py +16 -0
- autoencodix/modeling/.ipynb_checkpoints/_captum_forward-checkpoint.py +16 -0
- autoencodix/modeling/__init__.py +20 -0
- autoencodix/modeling/_captum_forward.py +20 -0
- autoencodix/modeling/_classifier.py +45 -0
- autoencodix/modeling/_imagevae_architecture.py +296 -0
- autoencodix/modeling/_imgfast_architecture.py +288 -0
- autoencodix/modeling/_layer_factory.py +71 -0
- autoencodix/modeling/_maskix_architecture.py +190 -0
- autoencodix/modeling/_ontix_architecture.py +305 -0
- autoencodix/modeling/_vanillix_architecture.py +136 -0
- autoencodix/modeling/_varix_architecture.py +192 -0
- autoencodix/notebooks/__init__.py +0 -0
- autoencodix/ontix.py +136 -0
- autoencodix/py.typed +0 -0
- autoencodix/stackix.py +122 -0
- autoencodix/trainers/__init__.py +6 -0
- autoencodix/trainers/_general_trainer.py +525 -0
- autoencodix/trainers/_maskix_trainer.py +116 -0
- autoencodix/trainers/_ontix_trainer.py +77 -0
- autoencodix/trainers/_stackix_orchestrator.py +522 -0
- autoencodix/trainers/_stackix_trainer.py +209 -0
- autoencodix/trainers/_xmodal_trainer.py +1006 -0
- autoencodix/utils/.ipynb_checkpoints/_explainer-checkpoint.py +118 -0
- autoencodix/utils/.ipynb_checkpoints/_model_output-checkpoint.py +30 -0
- autoencodix/utils/.ipynb_checkpoints/_utils-checkpoint.py +626 -0
- autoencodix/utils/.ipynb_checkpoints/adata_converter-checkpoint.py +99 -0
- autoencodix/utils/.ipynb_checkpoints/feature_importance-checkpoint.py +823 -0
- autoencodix/utils/__init__.py +22 -0
- autoencodix/utils/_annealer.py +193 -0
- autoencodix/utils/_bulkreader.py +232 -0
- autoencodix/utils/_explainer.py +246 -0
- autoencodix/utils/_imgreader.py +404 -0
- autoencodix/utils/_internals.py +23 -0
- autoencodix/utils/_llm_explainer.py +318 -0
- autoencodix/utils/_losses.py +20 -0
- autoencodix/utils/_model_output.py +30 -0
- autoencodix/utils/_result.py +393 -0
- autoencodix/utils/_screader.py +76 -0
- autoencodix/utils/_traindynamics.py +187 -0
- autoencodix/utils/_utils.py +690 -0
- autoencodix/utils/adata_converter.py +111 -0
- autoencodix/utils/example_data.py +398 -0
- autoencodix/utils/feature_importance.py +829 -0
- autoencodix/utils/prompts.py +21 -0
- autoencodix/vanillix.py +141 -0
- autoencodix/varix.py +80 -0
- autoencodix/visualize/__init__.py +5 -0
- autoencodix/visualize/_general_visualizer.py +913 -0
- autoencodix/visualize/_imagix_visualizer.py +111 -0
- autoencodix/visualize/_xmodal_visualizer.py +841 -0
- autoencodix/visualize/visualize.py +1214 -0
- autoencodix/xmodalix.py +318 -0
- autoencodix-0.2.0.dist-info/METADATA +143 -0
- autoencodix-0.2.0.dist-info/RECORD +91 -0
- autoencodix-0.2.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from ._layer_factory import LayerFactory
|
|
2
|
+
from ._vanillix_architecture import VanillixArchitecture
|
|
3
|
+
from ._varix_architecture import VarixArchitecture
|
|
4
|
+
from ._ontix_architecture import OntixArchitecture
|
|
5
|
+
from ._imagevae_architecture import ImageVAEArchitecture
|
|
6
|
+
from ._classifier import Classifier
|
|
7
|
+
from ._maskix_architecture import MaskixArchitectureVanilla
|
|
8
|
+
from ._imgfast_architecture import ImageVAEFastArchitecture
|
|
9
|
+
|
|
10
|
+
# test done
|
|
11
|
+
__all__ = [
|
|
12
|
+
"LayerFactory",
|
|
13
|
+
"VanillixArchitecture",
|
|
14
|
+
"VarixArchitecture",
|
|
15
|
+
"OntixArchitecture",
|
|
16
|
+
"ImageVAEArchitecture",
|
|
17
|
+
"Classifier",
|
|
18
|
+
"MaskixArchitectureVanilla",
|
|
19
|
+
"ImageVAEFastArchitecture",
|
|
20
|
+
]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
import torch.nn as nn
|
|
3
|
+
from autoencodix.base._base_autoencoder import BaseAutoencoder
|
|
4
|
+
from autoencodix.utils._model_output import ModelOutput
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class CaptumForward(nn.Module):
|
|
8
|
+
def __init__(self, model: BaseAutoencoder, dim: int):
|
|
9
|
+
super(CaptumForward, self).__init__() # <-- REQUIRED
|
|
10
|
+
self.model = model # (Registered as a submodule)
|
|
11
|
+
self.dim = dim
|
|
12
|
+
self.device = next(
|
|
13
|
+
model.parameters()
|
|
14
|
+
).device # Get the device of the model parameters
|
|
15
|
+
|
|
16
|
+
def forward(self, x: torch.Tensor):
|
|
17
|
+
mp: ModelOutput = self.model(x=x.to(self.device))
|
|
18
|
+
latent = mp.latentspace
|
|
19
|
+
output = latent[:, self.dim]
|
|
20
|
+
return output.unsqueeze(1).to("cpu")
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
import torch.nn as nn
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Classifier(nn.Module):
|
|
6
|
+
"""Multi-class classifier for adversarial training in n-modal latent space alignment.
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
Attributes:
|
|
10
|
+
input_dim: Dimension of the input features.
|
|
11
|
+
n_modalities: Number of modalities (classes) to classify.
|
|
12
|
+
n_hidden: Number of hidden units in the classifier network.
|
|
13
|
+
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def __init__(self, input_dim: int, n_modalities: int, n_hidden: int = 64) -> None:
|
|
17
|
+
super().__init__()
|
|
18
|
+
self.input_dim = input_dim
|
|
19
|
+
self.n_modalities = n_modalities
|
|
20
|
+
self.classifier = nn.Sequential(
|
|
21
|
+
nn.Linear(input_dim, n_hidden),
|
|
22
|
+
nn.ReLU(inplace=False),
|
|
23
|
+
nn.Dropout(0.1),
|
|
24
|
+
nn.Linear(n_hidden, n_hidden // 2),
|
|
25
|
+
nn.ReLU(inplace=False),
|
|
26
|
+
nn.Dropout(0.1),
|
|
27
|
+
nn.Linear(n_hidden // 2, n_modalities),
|
|
28
|
+
)
|
|
29
|
+
self.apply(self._init_weights)
|
|
30
|
+
|
|
31
|
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
32
|
+
"""Forward pass through the classifier.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
x: Input tensor of shape (batch_size, input_dim).
|
|
36
|
+
Returns:
|
|
37
|
+
Output tensor of shape (batch_size, n_modalities) representing class scores.
|
|
38
|
+
"""
|
|
39
|
+
return self.classifier(x)
|
|
40
|
+
|
|
41
|
+
def _init_weights(self, m: nn.Module) -> None:
|
|
42
|
+
if isinstance(m, nn.Linear):
|
|
43
|
+
nn.init.xavier_uniform_(m.weight)
|
|
44
|
+
if m.bias is not None:
|
|
45
|
+
m.bias.data.fill_(0.01)
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
import torch.nn as nn
|
|
3
|
+
from typing import Tuple, Optional, Union, Dict
|
|
4
|
+
from autoencodix.configs.default_config import DefaultConfig
|
|
5
|
+
from autoencodix.utils._model_output import ModelOutput
|
|
6
|
+
from autoencodix.base._base_autoencoder import BaseAutoencoder
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ImageVAEArchitecture(BaseAutoencoder):
|
|
10
|
+
"""This class defines a VAE, based on a CNN for images
|
|
11
|
+
|
|
12
|
+
It takes as input an image and of shape (C,W,H) and reconstructs it.
|
|
13
|
+
We ensure to have a latent space of shape <batchsize,1,LatentDim> and img_in.shape = img_out.shape
|
|
14
|
+
We have a fixed kernel_size=4, padding=1 and stride=2 (given from https://github.com/uhlerlab/cross-modal-auto_encoders/tree/master)
|
|
15
|
+
|
|
16
|
+
So we need to calculate how the image dimension changes after each Convolution (we assume W=H)
|
|
17
|
+
Applying the formular:
|
|
18
|
+
W_out = (((W - kernel_size + 2padding)/stride) + 1)
|
|
19
|
+
We get:
|
|
20
|
+
W_out = (((W-4+2*1)/2)+1) =
|
|
21
|
+
= (W-2/2)+1 =
|
|
22
|
+
= (2(0.5W-1)/2) +1 # factor 2 out
|
|
23
|
+
= 0.5W - 1 + 1
|
|
24
|
+
W_out = 0.5W
|
|
25
|
+
So in this configuration the output shape halfs after every convolutional step (assuming W=H)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
Attributes:
|
|
29
|
+
input_dim: (C,W,H) the input image shape
|
|
30
|
+
config: Configuration object containing model architecture parameters
|
|
31
|
+
_encoder: Encoder network of the autoencoder
|
|
32
|
+
_decoder: Decoder network of the autoencoder
|
|
33
|
+
latent_dim: Dimension of the latent space
|
|
34
|
+
nc: number of channels in the input image
|
|
35
|
+
h: height of the input image
|
|
36
|
+
w: width of the input image
|
|
37
|
+
img_shape: (C,W,H) the input image shape
|
|
38
|
+
hidden_dim: number of filters in the first convolutional layer
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
input_dim: Tuple[int, int, int], # (C,W,H) the input image shape
|
|
44
|
+
config: Optional[DefaultConfig],
|
|
45
|
+
ontologies: Optional[Union[Tuple, Dict]] = None,
|
|
46
|
+
feature_order: Optional[Union[Tuple, Dict]] = None,
|
|
47
|
+
# the input_dim is the number of channels in the image, e.g. 3
|
|
48
|
+
):
|
|
49
|
+
"""Initialize the ImageVAEArchitecture with the given configuration.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
input_dim: (C,W,H) the input image shape
|
|
53
|
+
config: Configuration object containing model parameters.
|
|
54
|
+
hidden_dim: number of filters in the first convolutional layer
|
|
55
|
+
"""
|
|
56
|
+
if config is None:
|
|
57
|
+
config = DefaultConfig()
|
|
58
|
+
self._config: DefaultConfig = config
|
|
59
|
+
super().__init__(config=config, input_dim=input_dim)
|
|
60
|
+
self.input_dim: int = input_dim
|
|
61
|
+
self.latent_dim: int = self._config.latent_dim
|
|
62
|
+
self.nc, self.h, self.w = input_dim
|
|
63
|
+
self.img_shape: Tuple[int, int, int] = input_dim
|
|
64
|
+
self.hidden_dim: int = self._config.hidden_dim
|
|
65
|
+
self._build_network()
|
|
66
|
+
self.apply(self._init_weights)
|
|
67
|
+
|
|
68
|
+
def _build_network(self):
|
|
69
|
+
"""Construct the encoder and decoder networks."""
|
|
70
|
+
self._encoder = nn.Sequential(
|
|
71
|
+
nn.Conv2d(
|
|
72
|
+
in_channels=self.nc,
|
|
73
|
+
out_channels=self.hidden_dim,
|
|
74
|
+
kernel_size=4,
|
|
75
|
+
stride=2,
|
|
76
|
+
padding=1,
|
|
77
|
+
bias=False,
|
|
78
|
+
),
|
|
79
|
+
nn.LeakyReLU(0.2, inplace=False),
|
|
80
|
+
nn.Conv2d(
|
|
81
|
+
in_channels=self.hidden_dim,
|
|
82
|
+
out_channels=self.hidden_dim * 2,
|
|
83
|
+
kernel_size=4,
|
|
84
|
+
stride=2,
|
|
85
|
+
padding=1,
|
|
86
|
+
bias=False,
|
|
87
|
+
),
|
|
88
|
+
nn.BatchNorm2d(self.hidden_dim * 2),
|
|
89
|
+
nn.LeakyReLU(0.2, inplace=False),
|
|
90
|
+
nn.Conv2d(
|
|
91
|
+
in_channels=self.hidden_dim * 2,
|
|
92
|
+
out_channels=self.hidden_dim * 4,
|
|
93
|
+
kernel_size=4,
|
|
94
|
+
stride=2,
|
|
95
|
+
padding=1,
|
|
96
|
+
bias=False,
|
|
97
|
+
),
|
|
98
|
+
nn.BatchNorm2d(self.hidden_dim * 4),
|
|
99
|
+
nn.LeakyReLU(0.2, inplace=False),
|
|
100
|
+
nn.Conv2d(
|
|
101
|
+
in_channels=self.hidden_dim * 4,
|
|
102
|
+
out_channels=self.hidden_dim * 8,
|
|
103
|
+
kernel_size=4,
|
|
104
|
+
stride=2,
|
|
105
|
+
padding=1,
|
|
106
|
+
bias=False,
|
|
107
|
+
),
|
|
108
|
+
nn.BatchNorm2d(self.hidden_dim * 8),
|
|
109
|
+
nn.LeakyReLU(0.2, inplace=False),
|
|
110
|
+
nn.Conv2d(
|
|
111
|
+
in_channels=self.hidden_dim * 8,
|
|
112
|
+
out_channels=self.hidden_dim * 8,
|
|
113
|
+
kernel_size=4,
|
|
114
|
+
stride=2,
|
|
115
|
+
padding=1,
|
|
116
|
+
bias=False,
|
|
117
|
+
),
|
|
118
|
+
nn.BatchNorm2d(self.hidden_dim * 8),
|
|
119
|
+
nn.LeakyReLU(0.2, inplace=False),
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
# to Calculate the image shape after the _encoder, we need to know the number of layers
|
|
123
|
+
# because the shape halfs after every Conv2D layer
|
|
124
|
+
self.num__encoder_layers = sum(
|
|
125
|
+
1 for _ in self._encoder.children() if isinstance(_, nn.Conv2d)
|
|
126
|
+
)
|
|
127
|
+
# So the output shape after all layers is in_shape / 2**N_layers
|
|
128
|
+
# We showed above in the DocString why the shape halfs
|
|
129
|
+
self.spatial_dim = self.h // (2**self.num__encoder_layers)
|
|
130
|
+
# In the Linear mu and logvar layer we need to flatten the 3D output to a 2D matrix
|
|
131
|
+
# Therefore we need to multiply the size of every out diemension of the input layer to the Linear layers
|
|
132
|
+
# This is hidden_dim * 8 (the number of filter/channel layer) * spatial dim (the widht of the image) * spatial diem (the height of the image)
|
|
133
|
+
# assuimg width = height
|
|
134
|
+
# The original paper had a fixed spatial dimension of 2, which only worked for images with 64x64 shape
|
|
135
|
+
self.mu = nn.Linear(
|
|
136
|
+
self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim, self.latent_dim
|
|
137
|
+
)
|
|
138
|
+
self.logvar = nn.Linear(
|
|
139
|
+
self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim, self.latent_dim
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
# the same logic goes for the first _decoder layer, which takes the latent_dim as inshape
|
|
143
|
+
# which is the outshape of the previous mu/logvar layer
|
|
144
|
+
# and the shape of the first ConvTranspose2D layer is the last outpus shape of the _encoder layer
|
|
145
|
+
# This the same multiplication as above
|
|
146
|
+
self.d1 = nn.Sequential(
|
|
147
|
+
nn.Linear(
|
|
148
|
+
self.latent_dim,
|
|
149
|
+
self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim,
|
|
150
|
+
),
|
|
151
|
+
nn.ReLU(inplace=False),
|
|
152
|
+
)
|
|
153
|
+
self._decoder = nn.Sequential(
|
|
154
|
+
nn.ConvTranspose2d(
|
|
155
|
+
in_channels=self.hidden_dim * 8,
|
|
156
|
+
out_channels=self.hidden_dim * 8,
|
|
157
|
+
kernel_size=4,
|
|
158
|
+
stride=2,
|
|
159
|
+
padding=1,
|
|
160
|
+
bias=False,
|
|
161
|
+
),
|
|
162
|
+
nn.BatchNorm2d(self.hidden_dim * 8),
|
|
163
|
+
nn.LeakyReLU(0.2, inplace=False),
|
|
164
|
+
nn.ConvTranspose2d(
|
|
165
|
+
in_channels=self.hidden_dim * 8,
|
|
166
|
+
out_channels=self.hidden_dim * 4,
|
|
167
|
+
kernel_size=4,
|
|
168
|
+
stride=2,
|
|
169
|
+
padding=1,
|
|
170
|
+
bias=False,
|
|
171
|
+
),
|
|
172
|
+
nn.BatchNorm2d(self.hidden_dim * 4),
|
|
173
|
+
nn.LeakyReLU(0.2, inplace=False),
|
|
174
|
+
nn.ConvTranspose2d(
|
|
175
|
+
in_channels=self.hidden_dim * 4,
|
|
176
|
+
out_channels=self.hidden_dim * 2,
|
|
177
|
+
kernel_size=4,
|
|
178
|
+
stride=2,
|
|
179
|
+
padding=1,
|
|
180
|
+
bias=False,
|
|
181
|
+
),
|
|
182
|
+
nn.BatchNorm2d(self.hidden_dim * 2),
|
|
183
|
+
nn.LeakyReLU(0.2, inplace=False),
|
|
184
|
+
nn.ConvTranspose2d(
|
|
185
|
+
in_channels=self.hidden_dim * 2,
|
|
186
|
+
out_channels=self.hidden_dim,
|
|
187
|
+
kernel_size=4,
|
|
188
|
+
stride=2,
|
|
189
|
+
padding=1,
|
|
190
|
+
bias=False,
|
|
191
|
+
),
|
|
192
|
+
nn.BatchNorm2d(self.hidden_dim),
|
|
193
|
+
nn.LeakyReLU(0.2, inplace=False),
|
|
194
|
+
nn.ConvTranspose2d(
|
|
195
|
+
in_channels=self.hidden_dim,
|
|
196
|
+
out_channels=self.nc,
|
|
197
|
+
kernel_size=4,
|
|
198
|
+
stride=2,
|
|
199
|
+
padding=1,
|
|
200
|
+
bias=False,
|
|
201
|
+
),
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
def _get_spatial_dim(self) -> int:
|
|
205
|
+
return self.spatial_dim
|
|
206
|
+
|
|
207
|
+
def encode(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
208
|
+
"""Encodes the input tensor x.
|
|
209
|
+
|
|
210
|
+
Args:
|
|
211
|
+
x: Input tensor
|
|
212
|
+
Returns:
|
|
213
|
+
The encoded latent space representation, or mu and logvar for VAEs.
|
|
214
|
+
|
|
215
|
+
"""
|
|
216
|
+
h = self._encoder(x)
|
|
217
|
+
# this makes sure we get the <batchsize, 1, latent_dim> shape for our latent space in the next step
|
|
218
|
+
# because we put all dimensionaltiy in the second dimension of the output shape.
|
|
219
|
+
# By covering all dimensionality here, we are sure that the rest is
|
|
220
|
+
h = h.view(-1, self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim)
|
|
221
|
+
logvar = self.logvar(h)
|
|
222
|
+
mu = self.mu(h)
|
|
223
|
+
# prevent mu and logvar from being too close to zero, this increased
|
|
224
|
+
# numerical stability
|
|
225
|
+
logvar = torch.clamp(logvar, 0.1, 20)
|
|
226
|
+
# replace mu when mu < 0.00000001 with 0.1
|
|
227
|
+
mu = torch.where(mu < 0.000001, torch.zeros_like(mu), mu)
|
|
228
|
+
return mu, logvar
|
|
229
|
+
|
|
230
|
+
def reparameterize(self, mu: torch.Tensor, logvar: torch.Tensor) -> torch.Tensor:
|
|
231
|
+
"""Reparameterization trick for VAE.
|
|
232
|
+
|
|
233
|
+
Args:
|
|
234
|
+
mu: mean of the latent distribution
|
|
235
|
+
logvar: log-variance of the latent distribution
|
|
236
|
+
Returns:
|
|
237
|
+
z: sampled latent vector
|
|
238
|
+
"""
|
|
239
|
+
std = torch.exp(0.5 * logvar)
|
|
240
|
+
eps = torch.randn_like(std)
|
|
241
|
+
return mu + eps * std
|
|
242
|
+
|
|
243
|
+
def get_latent_space(self, x: torch.Tensor) -> torch.Tensor:
|
|
244
|
+
"""Returns the latent space representation of the input.
|
|
245
|
+
|
|
246
|
+
Args:
|
|
247
|
+
x: Input tensor
|
|
248
|
+
Returns:
|
|
249
|
+
Latent space representation
|
|
250
|
+
|
|
251
|
+
"""
|
|
252
|
+
mu, logvar = self.encode(x)
|
|
253
|
+
return self.reparameterize(mu, logvar)
|
|
254
|
+
|
|
255
|
+
def decode(self, x: torch.Tensor) -> torch.Tensor:
|
|
256
|
+
"""Decode the latent tensor x
|
|
257
|
+
Args:
|
|
258
|
+
x: Latent tensor
|
|
259
|
+
Returns:
|
|
260
|
+
Decoded tensor, reconstructed from the latent space
|
|
261
|
+
"""
|
|
262
|
+
h = self.d1(x)
|
|
263
|
+
# here we do a similar thing as in the _encoder,
|
|
264
|
+
# but instead of ensuring the correct dimension for the latent space,
|
|
265
|
+
# we ensure the correct dimension for the first Conv2DTranspose layer
|
|
266
|
+
# so we make sure that the last 3 dimension are (n_filters, reduced_img_dim, reduced_img_dim)
|
|
267
|
+
h = h.view(-1, self.hidden_dim * 8, self.spatial_dim, self.spatial_dim)
|
|
268
|
+
return self._decoder(h)
|
|
269
|
+
|
|
270
|
+
def translate(self, z: torch.Tensor) -> torch.Tensor:
|
|
271
|
+
"""Reshapes the output to get actual images
|
|
272
|
+
|
|
273
|
+
Args:
|
|
274
|
+
z: Latent tensor
|
|
275
|
+
Returns:
|
|
276
|
+
Reconstructed image of shape (C,W,H)
|
|
277
|
+
"""
|
|
278
|
+
out = self.decode(z)
|
|
279
|
+
return out.view(-1, *self.img_shape)
|
|
280
|
+
|
|
281
|
+
def forward(self, x: torch.Tensor) -> ModelOutput:
|
|
282
|
+
"""Forward pass of the model.
|
|
283
|
+
Args:
|
|
284
|
+
x: Input tensor
|
|
285
|
+
Returns:
|
|
286
|
+
ModelOutput object containing the reconstructed tensor and latent tensor
|
|
287
|
+
"""
|
|
288
|
+
mu, logvar = self.encode(x)
|
|
289
|
+
z = self.reparameterize(mu, logvar)
|
|
290
|
+
return ModelOutput(
|
|
291
|
+
reconstruction=self.translate(z),
|
|
292
|
+
latentspace=z,
|
|
293
|
+
latent_mean=mu,
|
|
294
|
+
latent_logvar=logvar,
|
|
295
|
+
additional_info=None,
|
|
296
|
+
)
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
import torch.nn as nn
|
|
3
|
+
from typing import Tuple, Optional, Union, Dict
|
|
4
|
+
from autoencodix.configs.default_config import DefaultConfig
|
|
5
|
+
from autoencodix.utils._model_output import ModelOutput
|
|
6
|
+
from autoencodix.base._base_autoencoder import BaseAutoencoder
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ImageVAEFastArchitecture(BaseAutoencoder):
|
|
10
|
+
"""This class defines a VAE, based on a CNN for images
|
|
11
|
+
|
|
12
|
+
It takes as input an image and of shape (C,W,H) and reconstructs it.
|
|
13
|
+
We ensure to have a latent space of shape <batchsize,1,LatentDim> and img_in.shape = img_out.shape
|
|
14
|
+
We have a fixed kernel_size=4, padding=1 and stride=2 (given from https://github.com/uhlerlab/cross-modal-auto_encoders/tree/master)
|
|
15
|
+
|
|
16
|
+
So we need to calculate how the image dimension changes after each Convolution (we assume W=H)
|
|
17
|
+
Applying the formular:
|
|
18
|
+
W_out = (((W - kernel_size + 2padding)/stride) + 1)
|
|
19
|
+
We get:
|
|
20
|
+
W_out = (((W-4+2*1)/2)+1) =
|
|
21
|
+
= (W-2/2)+1 =
|
|
22
|
+
= (2(0.5W-1)/2) +1 # factor 2 out
|
|
23
|
+
= 0.5W - 1 + 1
|
|
24
|
+
W_out = 0.5W
|
|
25
|
+
So in this configuration the output shape halfs after every convolutional step (assuming W=H)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
Attributes:
|
|
29
|
+
input_dim: (C,W,H) the input image shape
|
|
30
|
+
config: Configuration object containing model architecture parameters
|
|
31
|
+
_encoder: Encoder network of the autoencoder
|
|
32
|
+
_decoder: Decoder network of the autoencoder
|
|
33
|
+
latent_dim: Dimension of the latent space
|
|
34
|
+
nc: number of channels in the input image
|
|
35
|
+
h: height of the input image
|
|
36
|
+
w: width of the input image
|
|
37
|
+
img_shape: (C,W,H) the input image shape
|
|
38
|
+
hidden_dim: number of filters in the first convolutional layer
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
input_dim: Tuple[int, int, int], # (C,W,H) the input image shape
|
|
44
|
+
config: Optional[DefaultConfig],
|
|
45
|
+
ontologies: Optional[Union[Tuple, Dict]] = None,
|
|
46
|
+
feature_order: Optional[Union[Tuple, Dict]] = None,
|
|
47
|
+
# the input_dim is the number of channels in the image, e.g. 3
|
|
48
|
+
):
|
|
49
|
+
"""Initialize the ImageVAEArchitecture with the given configuration.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
input_dim: (C,W,H) the input image shape
|
|
53
|
+
config: Configuration object containing model parameters.
|
|
54
|
+
hidden_dim: number of filters in the first convolutional layer
|
|
55
|
+
"""
|
|
56
|
+
if config is None:
|
|
57
|
+
config = DefaultConfig()
|
|
58
|
+
self._config: DefaultConfig = config
|
|
59
|
+
super().__init__(config=config, input_dim=input_dim)
|
|
60
|
+
self.input_dim: int = input_dim
|
|
61
|
+
self.latent_dim: int = self._config.latent_dim
|
|
62
|
+
self.nc, self.h, self.w = input_dim
|
|
63
|
+
self.img_shape: Tuple[int, int, int] = input_dim
|
|
64
|
+
self.hidden_dim: int = self._config.hidden_dim
|
|
65
|
+
self._build_network()
|
|
66
|
+
self.apply(self._init_weights)
|
|
67
|
+
|
|
68
|
+
def _build_network(self):
|
|
69
|
+
"""Construct the encoder and decoder networks."""
|
|
70
|
+
self._encoder = nn.Sequential(
|
|
71
|
+
nn.Conv2d(
|
|
72
|
+
in_channels=self.nc,
|
|
73
|
+
out_channels=self.hidden_dim,
|
|
74
|
+
kernel_size=4,
|
|
75
|
+
stride=2,
|
|
76
|
+
padding=1,
|
|
77
|
+
bias=False,
|
|
78
|
+
),
|
|
79
|
+
nn.LeakyReLU(0.2, inplace=True),
|
|
80
|
+
nn.Conv2d(
|
|
81
|
+
in_channels=self.hidden_dim,
|
|
82
|
+
out_channels=self.hidden_dim * 2,
|
|
83
|
+
kernel_size=4,
|
|
84
|
+
stride=2,
|
|
85
|
+
padding=1,
|
|
86
|
+
bias=False,
|
|
87
|
+
),
|
|
88
|
+
nn.LeakyReLU(0.2, inplace=True),
|
|
89
|
+
nn.Conv2d(
|
|
90
|
+
in_channels=self.hidden_dim * 2,
|
|
91
|
+
out_channels=self.hidden_dim * 4,
|
|
92
|
+
kernel_size=4,
|
|
93
|
+
stride=2,
|
|
94
|
+
padding=1,
|
|
95
|
+
bias=False,
|
|
96
|
+
),
|
|
97
|
+
nn.LeakyReLU(0.2, inplace=True),
|
|
98
|
+
nn.Conv2d(
|
|
99
|
+
in_channels=self.hidden_dim * 4,
|
|
100
|
+
out_channels=self.hidden_dim * 8,
|
|
101
|
+
kernel_size=4,
|
|
102
|
+
stride=2,
|
|
103
|
+
padding=1,
|
|
104
|
+
bias=False,
|
|
105
|
+
),
|
|
106
|
+
nn.LeakyReLU(0.2, inplace=True),
|
|
107
|
+
nn.Conv2d(
|
|
108
|
+
in_channels=self.hidden_dim * 8,
|
|
109
|
+
out_channels=self.hidden_dim * 8,
|
|
110
|
+
kernel_size=4,
|
|
111
|
+
stride=2,
|
|
112
|
+
padding=1,
|
|
113
|
+
bias=False,
|
|
114
|
+
),
|
|
115
|
+
nn.LeakyReLU(0.2, inplace=True),
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
# to Calculate the image shape after the _encoder, we need to know the number of layers
|
|
119
|
+
# because the shape halfs after every Conv2D layer
|
|
120
|
+
self.num__encoder_layers = sum(
|
|
121
|
+
1 for _ in self._encoder.children() if isinstance(_, nn.Conv2d)
|
|
122
|
+
)
|
|
123
|
+
# So the output shape after all layers is in_shape / 2**N_layers
|
|
124
|
+
# We showed above in the DocString why the shape halfs
|
|
125
|
+
self.spatial_dim = self.h // (2**self.num__encoder_layers)
|
|
126
|
+
# In the Linear mu and logvar layer we need to flatten the 3D output to a 2D matrix
|
|
127
|
+
# Therefore we need to multiply the size of every out diemension of the input layer to the Linear layers
|
|
128
|
+
# This is hidden_dim * 8 (the number of filter/channel layer) * spatial dim (the widht of the image) * spatial diem (the height of the image)
|
|
129
|
+
# assuimg width = height
|
|
130
|
+
# The original paper had a fixed spatial dimension of 2, which only worked for images with 64x64 shape
|
|
131
|
+
self.mu = nn.Linear(
|
|
132
|
+
self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim, self.latent_dim
|
|
133
|
+
)
|
|
134
|
+
self.logvar = nn.Linear(
|
|
135
|
+
self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim, self.latent_dim
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# the same logic goes for the first _decoder layer, which takes the latent_dim as inshape
|
|
139
|
+
# which is the outshape of the previous mu/logvar layer
|
|
140
|
+
# and the shape of the first ConvTranspose2D layer is the last outpus shape of the _encoder layer
|
|
141
|
+
# This the same multiplication as above
|
|
142
|
+
self.d1 = nn.Sequential(
|
|
143
|
+
nn.Linear(
|
|
144
|
+
self.latent_dim,
|
|
145
|
+
self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim,
|
|
146
|
+
),
|
|
147
|
+
nn.ReLU(inplace=False),
|
|
148
|
+
)
|
|
149
|
+
self._decoder = nn.Sequential(
|
|
150
|
+
nn.ConvTranspose2d(
|
|
151
|
+
in_channels=self.hidden_dim * 8,
|
|
152
|
+
out_channels=self.hidden_dim * 8,
|
|
153
|
+
kernel_size=4,
|
|
154
|
+
stride=2,
|
|
155
|
+
padding=1,
|
|
156
|
+
bias=False,
|
|
157
|
+
),
|
|
158
|
+
nn.LeakyReLU(0.2, inplace=True),
|
|
159
|
+
nn.ConvTranspose2d(
|
|
160
|
+
in_channels=self.hidden_dim * 8,
|
|
161
|
+
out_channels=self.hidden_dim * 4,
|
|
162
|
+
kernel_size=4,
|
|
163
|
+
stride=2,
|
|
164
|
+
padding=1,
|
|
165
|
+
bias=False,
|
|
166
|
+
),
|
|
167
|
+
nn.LeakyReLU(0.2, inplace=True),
|
|
168
|
+
nn.ConvTranspose2d(
|
|
169
|
+
in_channels=self.hidden_dim * 4,
|
|
170
|
+
out_channels=self.hidden_dim * 2,
|
|
171
|
+
kernel_size=4,
|
|
172
|
+
stride=2,
|
|
173
|
+
padding=1,
|
|
174
|
+
bias=False,
|
|
175
|
+
),
|
|
176
|
+
nn.LeakyReLU(0.2, inplace=True),
|
|
177
|
+
nn.ConvTranspose2d(
|
|
178
|
+
in_channels=self.hidden_dim * 2,
|
|
179
|
+
out_channels=self.hidden_dim,
|
|
180
|
+
kernel_size=4,
|
|
181
|
+
stride=2,
|
|
182
|
+
padding=1,
|
|
183
|
+
bias=False,
|
|
184
|
+
),
|
|
185
|
+
nn.LeakyReLU(0.2, inplace=True),
|
|
186
|
+
nn.ConvTranspose2d(
|
|
187
|
+
in_channels=self.hidden_dim,
|
|
188
|
+
out_channels=self.nc,
|
|
189
|
+
kernel_size=4,
|
|
190
|
+
stride=2,
|
|
191
|
+
padding=1,
|
|
192
|
+
bias=False,
|
|
193
|
+
),
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
def _get_spatial_dim(self) -> int:
|
|
197
|
+
return self.spatial_dim
|
|
198
|
+
|
|
199
|
+
def encode(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
200
|
+
"""Encodes the input tensor x.
|
|
201
|
+
|
|
202
|
+
Args:
|
|
203
|
+
x: Input tensor
|
|
204
|
+
Returns:
|
|
205
|
+
The encoded latent space representation, or mu and logvar for VAEs.
|
|
206
|
+
|
|
207
|
+
"""
|
|
208
|
+
h = self._encoder(x)
|
|
209
|
+
# this makes sure we get the <batchsize, 1, latent_dim> shape for our latent space in the next step
|
|
210
|
+
# because we put all dimensionaltiy in the second dimension of the output shape.
|
|
211
|
+
# By covering all dimensionality here, we are sure that the rest is
|
|
212
|
+
h = h.view(-1, self.hidden_dim * 8 * self.spatial_dim * self.spatial_dim)
|
|
213
|
+
logvar = self.logvar(h)
|
|
214
|
+
mu = self.mu(h)
|
|
215
|
+
# prevent mu and logvar from being too close to zero, this increased
|
|
216
|
+
# numerical stability
|
|
217
|
+
logvar = torch.clamp(logvar, 0.1, 20)
|
|
218
|
+
# replace mu when mu < 0.00000001 with 0.1
|
|
219
|
+
mu = torch.where(mu < 0.000001, torch.zeros_like(mu), mu)
|
|
220
|
+
return mu, logvar
|
|
221
|
+
|
|
222
|
+
def reparameterize(self, mu: torch.Tensor, logvar: torch.Tensor) -> torch.Tensor:
|
|
223
|
+
"""Reparameterization trick for VAE.
|
|
224
|
+
|
|
225
|
+
Args:
|
|
226
|
+
mu: mean of the latent distribution
|
|
227
|
+
logvar: log-variance of the latent distribution
|
|
228
|
+
Returns:
|
|
229
|
+
z: sampled latent vector
|
|
230
|
+
"""
|
|
231
|
+
std = torch.exp(0.5 * logvar)
|
|
232
|
+
eps = torch.randn_like(std)
|
|
233
|
+
return mu + eps * std
|
|
234
|
+
|
|
235
|
+
def get_latent_space(self, x: torch.Tensor) -> torch.Tensor:
|
|
236
|
+
"""Returns the latent space representation of the input.
|
|
237
|
+
|
|
238
|
+
Args:
|
|
239
|
+
x: Input tensor
|
|
240
|
+
Returns:
|
|
241
|
+
Latent space representation
|
|
242
|
+
|
|
243
|
+
"""
|
|
244
|
+
mu, logvar = self.encode(x)
|
|
245
|
+
return self.reparameterize(mu, logvar)
|
|
246
|
+
|
|
247
|
+
def decode(self, x: torch.Tensor) -> torch.Tensor:
|
|
248
|
+
"""Decode the latent tensor x
|
|
249
|
+
Args:
|
|
250
|
+
x: Latent tensor
|
|
251
|
+
Returns:
|
|
252
|
+
Decoded tensor, reconstructed from the latent space
|
|
253
|
+
"""
|
|
254
|
+
h = self.d1(x)
|
|
255
|
+
# here we do a similar thing as in the _encoder,
|
|
256
|
+
# but instead of ensuring the correct dimension for the latent space,
|
|
257
|
+
# we ensure the correct dimension for the first Conv2DTranspose layer
|
|
258
|
+
# so we make sure that the last 3 dimension are (n_filters, reduced_img_dim, reduced_img_dim)
|
|
259
|
+
h = h.view(-1, self.hidden_dim * 8, self.spatial_dim, self.spatial_dim)
|
|
260
|
+
return self._decoder(h)
|
|
261
|
+
|
|
262
|
+
def translate(self, z: torch.Tensor) -> torch.Tensor:
|
|
263
|
+
"""Reshapes the output to get actual images
|
|
264
|
+
|
|
265
|
+
Args:
|
|
266
|
+
z: Latent tensor
|
|
267
|
+
Returns:
|
|
268
|
+
Reconstructed image of shape (C,W,H)
|
|
269
|
+
"""
|
|
270
|
+
out = self.decode(z)
|
|
271
|
+
return out.view(-1, *self.img_shape)
|
|
272
|
+
|
|
273
|
+
def forward(self, x: torch.Tensor) -> ModelOutput:
|
|
274
|
+
"""Forward pass of the model.
|
|
275
|
+
Args:
|
|
276
|
+
x: Input tensor
|
|
277
|
+
Returns:
|
|
278
|
+
ModelOutput object containing the reconstructed tensor and latent tensor
|
|
279
|
+
"""
|
|
280
|
+
mu, logvar = self.encode(x)
|
|
281
|
+
z = self.reparameterize(mu, logvar)
|
|
282
|
+
return ModelOutput(
|
|
283
|
+
reconstruction=self.translate(z),
|
|
284
|
+
latentspace=z,
|
|
285
|
+
latent_mean=mu,
|
|
286
|
+
latent_logvar=logvar,
|
|
287
|
+
additional_info=None,
|
|
288
|
+
)
|