mccvae 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mccvae/__init__.py +5 -0
- mccvae/agent.py +288 -0
- mccvae/distributions/EuclideanNormal/__init__.py +15 -0
- mccvae/distributions/EuclideanNormal/arguments.py +3 -0
- mccvae/distributions/EuclideanNormal/distribution.py +26 -0
- mccvae/distributions/EuclideanNormal/layers.py +35 -0
- mccvae/distributions/EuclideanNormal/prior.py +18 -0
- mccvae/distributions/HWNormal/__init__.py +15 -0
- mccvae/distributions/HWNormal/arguments.py +3 -0
- mccvae/distributions/HWNormal/distribution.py +56 -0
- mccvae/distributions/HWNormal/layers.py +44 -0
- mccvae/distributions/HWNormal/prior.py +24 -0
- mccvae/distributions/LearnablePGMNormal/__init__.py +24 -0
- mccvae/distributions/LearnablePGMNormal/arguments.py +4 -0
- mccvae/distributions/LearnablePGMNormal/distribution.py +81 -0
- mccvae/distributions/LearnablePGMNormal/layers.py +218 -0
- mccvae/distributions/LearnablePGMNormal/prior.py +18 -0
- mccvae/distributions/PGMNormal/__init__.py +24 -0
- mccvae/distributions/PGMNormal/arguments.py +4 -0
- mccvae/distributions/PGMNormal/distribution.py +71 -0
- mccvae/distributions/PGMNormal/layers.py +82 -0
- mccvae/distributions/PGMNormal/prior.py +18 -0
- mccvae/distributions/PoincareNormal/__init__.py +16 -0
- mccvae/distributions/PoincareNormal/arguments.py +4 -0
- mccvae/distributions/PoincareNormal/ars.py +133 -0
- mccvae/distributions/PoincareNormal/distribution.py +58 -0
- mccvae/distributions/PoincareNormal/hyperbolic_radius.py +371 -0
- mccvae/distributions/PoincareNormal/hyperbolic_uniform.py +43 -0
- mccvae/distributions/PoincareNormal/layers.py +48 -0
- mccvae/distributions/PoincareNormal/prior.py +18 -0
- mccvae/distributions/__init__.py +0 -0
- mccvae/distributions/utils.py +21 -0
- mccvae/environment.py +361 -0
- mccvae/mixin.py +671 -0
- mccvae/model.py +480 -0
- mccvae/module.py +519 -0
- mccvae/palantir/__init__.py +28 -0
- mccvae/palantir/core.py +289 -0
- mccvae/palantir/diffusion.py +254 -0
- mccvae/palantir/magic.py +50 -0
- mccvae/palantir/pseudotime.py +431 -0
- mccvae/palantir/test_palantir.py +140 -0
- mccvae/palantir/utils.py +193 -0
- mccvae/unified_models/__init__.py +67 -0
- mccvae/unified_models/base_model.py +361 -0
- mccvae/unified_models/cellblast_model.py +349 -0
- mccvae/unified_models/clear_model.py +258 -0
- mccvae/unified_models/gmvae_model.py +560 -0
- mccvae/unified_models/scagcl_model.py +486 -0
- mccvae/unified_models/scalex_model.py +352 -0
- mccvae/unified_models/scdac_model.py +269 -0
- mccvae/unified_models/scdeepcluster_model.py +300 -0
- mccvae/unified_models/scdhmap_model.py +454 -0
- mccvae/unified_models/scdiffusion_model.py +479 -0
- mccvae/unified_models/scgcc_model.py +374 -0
- mccvae/unified_models/scgnn_model.py +210 -0
- mccvae/unified_models/scgpcl_model.py +665 -0
- mccvae/unified_models/schsc_model.py +626 -0
- mccvae/unified_models/scsimgcl_model.py +471 -0
- mccvae/unified_models/scsmd_model.py +387 -0
- mccvae/unified_models/sivae_model.py +460 -0
- mccvae/utils.py +228 -0
- mccvae-0.0.1.dist-info/METADATA +185 -0
- mccvae-0.0.1.dist-info/RECORD +67 -0
- mccvae-0.0.1.dist-info/WHEEL +5 -0
- mccvae-0.0.1.dist-info/licenses/LICENSE +21 -0
- mccvae-0.0.1.dist-info/top_level.txt +1 -0
mccvae/__init__.py
ADDED
mccvae/agent.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
|
|
2
|
+
"""
|
|
3
|
+
Agent Class for MCCVAE Training and Analysis
|
|
4
|
+
|
|
5
|
+
This module provides a high-level interface for training and analyzing MCCVAE models.
|
|
6
|
+
It includes comprehensive functionality for:
|
|
7
|
+
- Model training with progress tracking
|
|
8
|
+
- Latent representation extraction for biological correlation learning
|
|
9
|
+
- Information bottleneck analysis for coupled representations
|
|
10
|
+
- Visualization and analysis utilities for single-cell genomics
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from .environment import Env
|
|
14
|
+
from anndata import AnnData
|
|
15
|
+
import torch
|
|
16
|
+
import tqdm
|
|
17
|
+
from typing import Literal
|
|
18
|
+
import numpy as np
|
|
19
|
+
import warnings
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Agent(Env):
|
|
24
|
+
"""
|
|
25
|
+
High-level agent for MCCVAE model training and analysis.
|
|
26
|
+
|
|
27
|
+
This class provides a comprehensive interface for training MoCo Coupling Variational
|
|
28
|
+
Autoencoders with contrastive learning capabilities designed for single-cell genomics.
|
|
29
|
+
It offers methods for:
|
|
30
|
+
|
|
31
|
+
- Model training with automatic progress tracking
|
|
32
|
+
- Latent representation and embedding extraction
|
|
33
|
+
- Information bottleneck analysis for coupled biological representations
|
|
34
|
+
- Visualization-ready outputs for single-cell analysis
|
|
35
|
+
|
|
36
|
+
The agent integrates seamlessly with AnnData objects commonly used in
|
|
37
|
+
single-cell analysis and focuses on learning coupled latent representations
|
|
38
|
+
that capture biological correlations intrinsic to single-cell data.
|
|
39
|
+
|
|
40
|
+
The information bottleneck component specifically learns compressed
|
|
41
|
+
representations that preserve essential biological correlations while
|
|
42
|
+
filtering out technical noise, enabling better understanding of cellular
|
|
43
|
+
states and biological processes.
|
|
44
|
+
|
|
45
|
+
Parameters
|
|
46
|
+
----------
|
|
47
|
+
adata : AnnData
|
|
48
|
+
Annotated single-cell data object containing the dataset
|
|
49
|
+
layer : str, default="counts"
|
|
50
|
+
Layer name in adata to use for training data
|
|
51
|
+
percent : float, default=0.01
|
|
52
|
+
Fraction of dataset to use per batch (0.0 to 1.0)
|
|
53
|
+
recon : float, default=1.0
|
|
54
|
+
Weight for reconstruction loss
|
|
55
|
+
irecon : float, default=0.0
|
|
56
|
+
Weight for information bottleneck reconstruction loss
|
|
57
|
+
beta : float, default=1.0
|
|
58
|
+
Weight for KL divergence regularization (β-VAE)
|
|
59
|
+
dip : float, default=0.0
|
|
60
|
+
Weight for DIP regularization
|
|
61
|
+
tc : float, default=0.0
|
|
62
|
+
Weight for total correlation regularization
|
|
63
|
+
info : float, default=0.0
|
|
64
|
+
Weight for InfoVAE MMD regularization
|
|
65
|
+
hidden_dim : int, default=128
|
|
66
|
+
Dimension of hidden layers in the network
|
|
67
|
+
latent_dim : int, default=10
|
|
68
|
+
Dimension of the latent space
|
|
69
|
+
i_dim : int, default=2
|
|
70
|
+
Dimension of information bottleneck for coupled biological representations
|
|
71
|
+
use_moco : bool, default=False
|
|
72
|
+
Whether to enable Momentum Contrast learning
|
|
73
|
+
loss_mode : Literal["mse", "nb", "zinb"], default="nb"
|
|
74
|
+
Loss function mode for reconstruction
|
|
75
|
+
lr : float, default=1e-4
|
|
76
|
+
Learning rate for optimization
|
|
77
|
+
vae_reg : float, default=0.5
|
|
78
|
+
Regularization weight for VAE latent representations
|
|
79
|
+
moco_weight : float, default=1.0
|
|
80
|
+
Weight for contrastive learning loss
|
|
81
|
+
moco_T : float, default=0.2
|
|
82
|
+
Temperature parameter for MoCo contrastive loss
|
|
83
|
+
aug_prob : float, default=0.5
|
|
84
|
+
Probability of applying data augmentation
|
|
85
|
+
mask_prob : float, default=0.2
|
|
86
|
+
Probability of masking each gene during augmentation
|
|
87
|
+
noise_prob : float, default=0.7
|
|
88
|
+
Probability of adding noise to each gene during augmentation
|
|
89
|
+
use_qm : bool, default=True
|
|
90
|
+
Whether to use mean (True) or samples (False) for latent representations
|
|
91
|
+
device : torch.device
|
|
92
|
+
Computation device (automatically selects CUDA if available)
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
def __init__(
|
|
96
|
+
self,
|
|
97
|
+
adata: AnnData,
|
|
98
|
+
layer: str = "counts",
|
|
99
|
+
percent: float = 0.01,
|
|
100
|
+
recon: float = 1.0,
|
|
101
|
+
irecon: float = 0.0,
|
|
102
|
+
beta: float = 1.0,
|
|
103
|
+
dip: float = 0.0,
|
|
104
|
+
tc: float = 0.0,
|
|
105
|
+
info: float = 0.0,
|
|
106
|
+
hidden_dim: int = 128,
|
|
107
|
+
latent_dim: int = 10,
|
|
108
|
+
i_dim: int = 2,
|
|
109
|
+
use_moco: bool = False,
|
|
110
|
+
use_bn: bool = True,
|
|
111
|
+
loss_mode: Literal["mse", "nb", "zinb"] = "nb",
|
|
112
|
+
lr: float = 1e-4,
|
|
113
|
+
vae_reg: float = 0.5,
|
|
114
|
+
moco_weight: float = 1.0,
|
|
115
|
+
moco_T: float = 0.2,
|
|
116
|
+
aug_prob: float = 0.5,
|
|
117
|
+
mask_prob: float = 0.2,
|
|
118
|
+
noise_prob: float = 0.7,
|
|
119
|
+
use_qm: bool = True,
|
|
120
|
+
device: torch.device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu"),
|
|
121
|
+
):
|
|
122
|
+
# Initialize the parent environment with all parameters
|
|
123
|
+
super().__init__(
|
|
124
|
+
adata=adata,
|
|
125
|
+
layer=layer,
|
|
126
|
+
percent=percent,
|
|
127
|
+
recon=recon,
|
|
128
|
+
irecon=irecon,
|
|
129
|
+
beta=beta,
|
|
130
|
+
dip=dip,
|
|
131
|
+
tc=tc,
|
|
132
|
+
info=info,
|
|
133
|
+
hidden_dim=hidden_dim,
|
|
134
|
+
latent_dim=latent_dim,
|
|
135
|
+
i_dim=i_dim,
|
|
136
|
+
use_moco=use_moco,
|
|
137
|
+
use_bn=use_bn,
|
|
138
|
+
loss_mode=loss_mode,
|
|
139
|
+
lr=lr,
|
|
140
|
+
vae_reg=vae_reg,
|
|
141
|
+
moco_weight=moco_weight,
|
|
142
|
+
moco_T=moco_T,
|
|
143
|
+
aug_prob=aug_prob,
|
|
144
|
+
mask_prob=mask_prob,
|
|
145
|
+
noise_prob=noise_prob,
|
|
146
|
+
use_qm=use_qm,
|
|
147
|
+
device=device,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
def fit(self, epochs: int = 1000) -> 'Agent':
|
|
151
|
+
"""
|
|
152
|
+
Train the MCCVAE model with progress tracking.
|
|
153
|
+
|
|
154
|
+
This method performs the complete training loop with:
|
|
155
|
+
- Automatic batch sampling and data loading
|
|
156
|
+
- Progress bar with real-time metrics display
|
|
157
|
+
- Periodic evaluation metric computation
|
|
158
|
+
- Loss and performance tracking
|
|
159
|
+
- Learning of coupled biological representations
|
|
160
|
+
|
|
161
|
+
The training progress shows key metrics including:
|
|
162
|
+
- Total loss value
|
|
163
|
+
- ARI (Adjusted Rand Index) for clustering accuracy
|
|
164
|
+
- NMI (Normalized Mutual Information) for clustering quality
|
|
165
|
+
- ASW (Average Silhouette Width) for cluster separation
|
|
166
|
+
- C_H (Calinski-Harabasz Index) for cluster validity
|
|
167
|
+
- D_B (Davies-Bouldin Index) for cluster compactness
|
|
168
|
+
- P_C (Pearson Correlation) for disentanglement measure
|
|
169
|
+
|
|
170
|
+
Parameters
|
|
171
|
+
----------
|
|
172
|
+
epochs : int, default=1000
|
|
173
|
+
Number of training epochs to perform
|
|
174
|
+
|
|
175
|
+
Returns
|
|
176
|
+
-------
|
|
177
|
+
Agent
|
|
178
|
+
Self (for method chaining)
|
|
179
|
+
"""
|
|
180
|
+
with tqdm.tqdm(total=int(epochs), desc="Training MCCVAE", ncols=150) as progress_bar:
|
|
181
|
+
for epoch in range(int(epochs)):
|
|
182
|
+
# Load batch data (with augmentation if using MoCo)
|
|
183
|
+
primary_data, query_data, key_data = self.load_data()
|
|
184
|
+
|
|
185
|
+
# Perform training step
|
|
186
|
+
if self.use_moco:
|
|
187
|
+
self.step(primary_data, query_data, key_data)
|
|
188
|
+
else:
|
|
189
|
+
self.step(primary_data)
|
|
190
|
+
|
|
191
|
+
# Update progress display every 10 epochs
|
|
192
|
+
if (epoch + 1) % 10 == 0:
|
|
193
|
+
latest_loss = self.training_losses[-1]
|
|
194
|
+
latest_scores = self.evaluation_scores[-1]
|
|
195
|
+
|
|
196
|
+
progress_bar.set_postfix({
|
|
197
|
+
"Loss": f"{latest_loss[0]:.2f}",
|
|
198
|
+
"ARI": f"{latest_scores[0]:.2f}",
|
|
199
|
+
"NMI": f"{latest_scores[1]:.2f}",
|
|
200
|
+
"ASW": f"{latest_scores[2]:.2f}",
|
|
201
|
+
"C_H": f"{latest_scores[3]:.2f}",
|
|
202
|
+
"D_B": f"{latest_scores[4]:.2f}",
|
|
203
|
+
"P_C": f"{latest_scores[5]:.2f}",
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
progress_bar.update(1)
|
|
207
|
+
|
|
208
|
+
return self
|
|
209
|
+
|
|
210
|
+
def get_iembed(self) -> np.ndarray:
|
|
211
|
+
"""
|
|
212
|
+
Extract information bottleneck embeddings from the full dataset.
|
|
213
|
+
|
|
214
|
+
These embeddings represent the coupled latent representations learned
|
|
215
|
+
through the information bottleneck layer, specifically designed to capture
|
|
216
|
+
biological correlations intrinsic to single-cell data. They provide a
|
|
217
|
+
compressed view of the latent space that preserves essential biological
|
|
218
|
+
relationships while filtering out technical noise.
|
|
219
|
+
|
|
220
|
+
Returns
|
|
221
|
+
-------
|
|
222
|
+
np.ndarray
|
|
223
|
+
Information bottleneck embeddings with shape (n_cells, i_dim)
|
|
224
|
+
representing coupled biological correlations
|
|
225
|
+
"""
|
|
226
|
+
information_embeddings = self.take_iembed(self.dataset)
|
|
227
|
+
return information_embeddings
|
|
228
|
+
|
|
229
|
+
def get_latent(self) -> np.ndarray:
|
|
230
|
+
"""
|
|
231
|
+
Extract latent representations from the full dataset.
|
|
232
|
+
|
|
233
|
+
These are the primary latent space representations learned by the MCCVAE,
|
|
234
|
+
capturing the underlying biological structure and cellular relationships
|
|
235
|
+
in single-cell data. They provide a comprehensive view of cellular states
|
|
236
|
+
encoded in the gene expression patterns.
|
|
237
|
+
|
|
238
|
+
Returns
|
|
239
|
+
-------
|
|
240
|
+
np.ndarray
|
|
241
|
+
Latent representations with shape (n_cells, latent_dim)
|
|
242
|
+
"""
|
|
243
|
+
latent_representations = self.take_latent(self.dataset)
|
|
244
|
+
return latent_representations
|
|
245
|
+
|
|
246
|
+
def get_bottleneck(self) -> np.ndarray:
|
|
247
|
+
"""
|
|
248
|
+
Extract the compressed bottleneck embeddings (l_e) from the full dataset.
|
|
249
|
+
|
|
250
|
+
This is an alias for get_iembed() provided for consistency with the README
|
|
251
|
+
documentation. The bottleneck embeddings represent the compressed information
|
|
252
|
+
bottleneck layer that captures biological correlations.
|
|
253
|
+
|
|
254
|
+
Returns
|
|
255
|
+
-------
|
|
256
|
+
np.ndarray
|
|
257
|
+
Bottleneck embeddings with shape (n_cells, i_dim)
|
|
258
|
+
"""
|
|
259
|
+
return self.get_iembed()
|
|
260
|
+
|
|
261
|
+
def get_refined(self) -> np.ndarray:
|
|
262
|
+
"""
|
|
263
|
+
Extract the refined latent representations (l_d) from the full dataset.
|
|
264
|
+
|
|
265
|
+
The refined representations are obtained by passing the primary latent
|
|
266
|
+
representations through the information bottleneck:
|
|
267
|
+
1. Encoding data to latent space (z)
|
|
268
|
+
2. Compressing through bottleneck encoder to get l_e
|
|
269
|
+
3. Expanding through bottleneck decoder to get l_d
|
|
270
|
+
4. l_d is used for secondary reconstruction in the coupling component
|
|
271
|
+
|
|
272
|
+
These refined representations (l_d) have the same dimensionality as the
|
|
273
|
+
original latent space (latent_dim) but have been refined through the
|
|
274
|
+
information bottleneck, capturing essential biological correlations
|
|
275
|
+
while filtering out technical noise.
|
|
276
|
+
|
|
277
|
+
Returns
|
|
278
|
+
-------
|
|
279
|
+
np.ndarray
|
|
280
|
+
Refined latent representations (l_d) with shape (n_cells, latent_dim)
|
|
281
|
+
|
|
282
|
+
See Also
|
|
283
|
+
--------
|
|
284
|
+
get_iembed : Get information bottleneck embeddings (l_e) - the compressed representation
|
|
285
|
+
get_bottleneck : Alias for get_iembed()
|
|
286
|
+
get_latent : Get primary latent representations (z)
|
|
287
|
+
"""
|
|
288
|
+
return self.take_refined(self.dataset)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Euclidean Normal Distribution
|
|
3
|
+
Standard Gaussian distribution in Euclidean space
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from .distribution import Distribution
|
|
7
|
+
from .layers import VanillaEncoderLayer, VanillaDecoderLayer
|
|
8
|
+
from .prior import get_prior
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
'Distribution',
|
|
12
|
+
'VanillaEncoderLayer',
|
|
13
|
+
'VanillaDecoderLayer',
|
|
14
|
+
'get_prior',
|
|
15
|
+
]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
from torch.distributions import Normal
|
|
3
|
+
|
|
4
|
+
from ..utils import euclidean_kl_div
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Distribution():
|
|
8
|
+
def __init__(self, mean, logvar) -> None:
|
|
9
|
+
self.mean = mean
|
|
10
|
+
self.logvar = logvar
|
|
11
|
+
|
|
12
|
+
self.base = Normal(self.mean, (self.logvar * 0.5).exp())
|
|
13
|
+
|
|
14
|
+
def log_prob(self, z):
|
|
15
|
+
return self.base.log_prob(z)
|
|
16
|
+
|
|
17
|
+
def rsample(self, N):
|
|
18
|
+
return self.base.rsample([N])
|
|
19
|
+
|
|
20
|
+
def sample(self, N):
|
|
21
|
+
with torch.no_grad():
|
|
22
|
+
return self.rsample(N)
|
|
23
|
+
|
|
24
|
+
def kl_div(self, target_dist):
|
|
25
|
+
return euclidean_kl_div(self.mean, self.logvar, target_dist.mean, target_dist.logvar)
|
|
26
|
+
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
from torch import nn
|
|
3
|
+
from torch.nn import functional as F
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class VanillaEncoderLayer(nn.Module):
|
|
7
|
+
def __init__(self, args, feature_dim) -> None:
|
|
8
|
+
super().__init__()
|
|
9
|
+
|
|
10
|
+
self.latent_dim = int(args.latent_dim * 2)
|
|
11
|
+
self.feature_dim = feature_dim
|
|
12
|
+
|
|
13
|
+
self.variational = nn.Linear(
|
|
14
|
+
self.feature_dim,
|
|
15
|
+
2 * self.latent_dim
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
def forward(self, feature):
|
|
19
|
+
feature = self.variational(feature)
|
|
20
|
+
mean, logvar = torch.split(
|
|
21
|
+
feature,
|
|
22
|
+
[self.latent_dim, self.latent_dim],
|
|
23
|
+
dim=-1
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
return mean, logvar
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class VanillaDecoderLayer(nn.Module):
|
|
30
|
+
def __init__(self, args) -> None:
|
|
31
|
+
super().__init__()
|
|
32
|
+
|
|
33
|
+
def forward(self, z):
|
|
34
|
+
return z
|
|
35
|
+
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
|
|
3
|
+
from .distribution import Distribution
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_prior(args):
|
|
7
|
+
mean = torch.zeros(
|
|
8
|
+
[1, int(args.latent_dim * 2)],
|
|
9
|
+
device=args.device
|
|
10
|
+
)
|
|
11
|
+
covar = torch.zeros(
|
|
12
|
+
[1, int(args.latent_dim * 2)],
|
|
13
|
+
device=args.device
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
prior = Distribution(mean, covar)
|
|
17
|
+
return prior
|
|
18
|
+
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Hyperbolic Wrapped Normal Distribution
|
|
3
|
+
Wrapped normal distribution on hyperbolic space using Lorentz model
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from .distribution import Distribution
|
|
7
|
+
from .layers import VanillaEncoderLayer, VanillaDecoderLayer
|
|
8
|
+
from .prior import get_prior
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
'Distribution',
|
|
12
|
+
'VanillaEncoderLayer',
|
|
13
|
+
'VanillaDecoderLayer',
|
|
14
|
+
'get_prior',
|
|
15
|
+
]
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
import geoopt
|
|
3
|
+
from torch.nn import functional as F
|
|
4
|
+
from torch.distributions import Normal
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Distribution():
|
|
8
|
+
def __init__(self, mean, sigma) -> None:
|
|
9
|
+
self.mean = mean # (1, *, 3)
|
|
10
|
+
self.sigma = torch.nan_to_num(sigma, nan=1.0, posinf=1.0, neginf=1.0).clamp_min(1e-8)
|
|
11
|
+
|
|
12
|
+
self.latent_dim = 2
|
|
13
|
+
self.base = Normal(
|
|
14
|
+
torch.zeros([*self.sigma.shape[:-1], 2], device=self.mean.device),
|
|
15
|
+
self.sigma,
|
|
16
|
+
validate_args=False,
|
|
17
|
+
)
|
|
18
|
+
self.manifold = geoopt.manifolds.Lorentz()
|
|
19
|
+
self.origin = self.manifold.origin(
|
|
20
|
+
self.mean.size(),
|
|
21
|
+
device=self.mean.device
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
self.kl_div = None
|
|
25
|
+
|
|
26
|
+
def log_prob(self, z):
|
|
27
|
+
eps = 1e-8
|
|
28
|
+
|
|
29
|
+
u = self.manifold.logmap(self.mean, z)
|
|
30
|
+
u = torch.nan_to_num(u, nan=0.0, posinf=0.0, neginf=0.0)
|
|
31
|
+
|
|
32
|
+
v = self.manifold.transp(self.mean, self.origin, u)
|
|
33
|
+
v = torch.nan_to_num(v, nan=0.0, posinf=0.0, neginf=0.0)
|
|
34
|
+
|
|
35
|
+
v_spatial = v[..., 1:]
|
|
36
|
+
log_prob_v = self.base.log_prob(v_spatial).sum(dim=-1)
|
|
37
|
+
|
|
38
|
+
r = self.manifold.norm(u).clamp_min(eps)
|
|
39
|
+
ratio = (torch.sinh(r) / r).clamp_min(eps)
|
|
40
|
+
log_det = (self.latent_dim - 1) * torch.log(ratio)
|
|
41
|
+
|
|
42
|
+
return log_prob_v - log_det
|
|
43
|
+
|
|
44
|
+
def rsample(self, N):
|
|
45
|
+
v = self.base.rsample([N]) # (N, *, 2)
|
|
46
|
+
v = F.pad(v, (1, 0)) # (N, *, 3)
|
|
47
|
+
|
|
48
|
+
u = self.manifold.transp0(self.mean, v) # (N, *, 3)
|
|
49
|
+
z = self.manifold.expmap(self.mean, u)
|
|
50
|
+
|
|
51
|
+
return z
|
|
52
|
+
|
|
53
|
+
def sample(self, N):
|
|
54
|
+
with torch.no_grad():
|
|
55
|
+
return self.rsample(N)
|
|
56
|
+
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
import geoopt
|
|
3
|
+
from torch import nn
|
|
4
|
+
from torch.nn import functional as F
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class VanillaEncoderLayer(nn.Module):
|
|
8
|
+
def __init__(self, args, feature_dim) -> None:
|
|
9
|
+
super().__init__()
|
|
10
|
+
|
|
11
|
+
self.latent_dim = args.latent_dim
|
|
12
|
+
self.feature_dim = feature_dim
|
|
13
|
+
|
|
14
|
+
self.variational = nn.Linear(
|
|
15
|
+
self.feature_dim,
|
|
16
|
+
4 * self.latent_dim
|
|
17
|
+
)
|
|
18
|
+
self.manifold = geoopt.manifolds.Lorentz()
|
|
19
|
+
|
|
20
|
+
def forward(self, feature):
|
|
21
|
+
feature = self.variational(feature)
|
|
22
|
+
mean, logsigma = torch.split(
|
|
23
|
+
feature,
|
|
24
|
+
[2 * self.latent_dim, 2 * self.latent_dim],
|
|
25
|
+
dim=-1
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
mean = mean.view(*mean.shape[:-1], self.latent_dim, 2)
|
|
29
|
+
mean = self.manifold.expmap0(F.pad(mean, (1, 0)))
|
|
30
|
+
sigma = F.softplus(logsigma).view(*logsigma.shape[:-1], self.latent_dim, 2)
|
|
31
|
+
|
|
32
|
+
return mean, sigma
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class VanillaDecoderLayer(nn.Module):
|
|
36
|
+
def __init__(self, args) -> None:
|
|
37
|
+
super().__init__()
|
|
38
|
+
self.manifold = geoopt.manifolds.Lorentz()
|
|
39
|
+
|
|
40
|
+
def forward(self, z):
|
|
41
|
+
z = self.manifold.logmap0(z)[..., 1:]
|
|
42
|
+
z = z.reshape(*z.shape[:-2], -1)
|
|
43
|
+
return z
|
|
44
|
+
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
import geoopt
|
|
3
|
+
from torch.nn import functional as F
|
|
4
|
+
|
|
5
|
+
from .distribution import Distribution
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def get_prior(args):
|
|
9
|
+
m = geoopt.manifolds.Lorentz()
|
|
10
|
+
|
|
11
|
+
mean = torch.zeros(
|
|
12
|
+
[1, args.latent_dim, 2],
|
|
13
|
+
device=args.device
|
|
14
|
+
)
|
|
15
|
+
mean = m.expmap0(F.pad(mean, (1, 0)))
|
|
16
|
+
|
|
17
|
+
sigma = torch.ones(
|
|
18
|
+
[1, args.latent_dim, 2],
|
|
19
|
+
device=args.device
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
prior = Distribution(mean, sigma)
|
|
23
|
+
return prior
|
|
24
|
+
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Learnable Pseudo-Gaussian Mixture Normal Distribution
|
|
3
|
+
PGM Normal with learnable concentration parameters and coordinate transformations
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from .distribution import Distribution
|
|
7
|
+
from .layers import (
|
|
8
|
+
EncoderLayer,
|
|
9
|
+
VanillaEncoderLayer,
|
|
10
|
+
ExpEncoderLayer,
|
|
11
|
+
VanillaDecoderLayer,
|
|
12
|
+
LogDecoderLayer,
|
|
13
|
+
)
|
|
14
|
+
from .prior import get_prior
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
'Distribution',
|
|
18
|
+
'EncoderLayer',
|
|
19
|
+
'VanillaEncoderLayer',
|
|
20
|
+
'ExpEncoderLayer',
|
|
21
|
+
'VanillaDecoderLayer',
|
|
22
|
+
'LogDecoderLayer',
|
|
23
|
+
'get_prior',
|
|
24
|
+
]
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
def add_distribution_args(parser):
|
|
2
|
+
group = parser.add_argument_group('LearnablePGMNormal')
|
|
3
|
+
group.add_argument('--encoder_layer', type=str, choices=['Vanilla', 'Exp'], default='Vanilla')
|
|
4
|
+
group.add_argument('--decoder_layer', type=str, choices=['Vanilla', 'Log'], default='Vanilla')
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
from math import log, sqrt, pi
|
|
3
|
+
from torch.distributions import Normal, Gamma
|
|
4
|
+
from torch.distributions.distribution import Distribution
|
|
5
|
+
from ..utils import euclidean_kl_div, gamma_kl_div
|
|
6
|
+
|
|
7
|
+
@torch.jit.script
|
|
8
|
+
def _log_prob(kl, log_gamma_square, log_beta_square, c):
|
|
9
|
+
log_prob = -kl / log_gamma_square.exp() / (2 * -c)
|
|
10
|
+
log_prob = log_prob - 1.5 * log_beta_square
|
|
11
|
+
log_prob = log_prob - 0.5 * log_gamma_square
|
|
12
|
+
|
|
13
|
+
gamma_factor = (-log_gamma_square).exp() / (4 * -c)
|
|
14
|
+
log_prob = log_prob - gamma_factor
|
|
15
|
+
log_prob = log_prob - torch.lgamma(gamma_factor)
|
|
16
|
+
log_prob = log_prob - gamma_factor * ((4 * -c).log() + log_gamma_square)
|
|
17
|
+
# log_prob = log_prob - 0.5 * log(2 * pi)
|
|
18
|
+
|
|
19
|
+
return log_prob
|
|
20
|
+
|
|
21
|
+
class Distribution(Distribution):
|
|
22
|
+
def __init__(self, means, log_gamma_square):
|
|
23
|
+
super().__init__()
|
|
24
|
+
# means: [..., 3] = (alpha, log_beta_square, log_c)
|
|
25
|
+
self.alpha = means[..., 0]
|
|
26
|
+
self.log_beta_square = means[..., 1]
|
|
27
|
+
# c is curvature magnitude (>0), underlying manifold curvature is -c
|
|
28
|
+
self.c = means[..., 2].exp()
|
|
29
|
+
self.log_gamma_square = log_gamma_square
|
|
30
|
+
|
|
31
|
+
# Normal base parameters
|
|
32
|
+
self.normal_mu = self.alpha # mean
|
|
33
|
+
self.normal_logvar = self.log_beta_square # log variance
|
|
34
|
+
|
|
35
|
+
# ---------- FIX: make Gamma parameters valid (both > 0) ----------
|
|
36
|
+
denom = 4.0 * (self.c + 1e-6) # strictly positive
|
|
37
|
+
self.gamma_a = (-self.log_gamma_square).exp() / denom + 1.0 # concentration > 0
|
|
38
|
+
self.gamma_b = (-self.normal_logvar).exp() / denom # rate > 0
|
|
39
|
+
|
|
40
|
+
# ---------- FIX: define base1 so rsample() works ----------
|
|
41
|
+
# Normal over alpha with variance exp(log_beta_square)
|
|
42
|
+
self.base1 = Normal(
|
|
43
|
+
loc=self.normal_mu,
|
|
44
|
+
scale=(0.5 * self.normal_logvar).exp() # std = exp(0.5 * logvar)
|
|
45
|
+
)
|
|
46
|
+
self.base2 = Gamma(self.gamma_a, self.gamma_b)
|
|
47
|
+
|
|
48
|
+
def log_prob(self, z):
|
|
49
|
+
target_mean, target_logvar = sqrt(-2 * self.c) * z[..., 0], z[..., 1]
|
|
50
|
+
kl = euclidean_kl_div(
|
|
51
|
+
target_mean,
|
|
52
|
+
target_logvar,
|
|
53
|
+
self.alpha * (-2 * self.c).sqrt(),
|
|
54
|
+
self.log_beta_square
|
|
55
|
+
)
|
|
56
|
+
return _log_prob(kl, self.log_gamma_square, self.log_beta_square, self.c)
|
|
57
|
+
|
|
58
|
+
def rsample(self, N):
|
|
59
|
+
sample_mean = self.base1.rsample([N])
|
|
60
|
+
sample_logvar = self.base2.rsample([N]).log()
|
|
61
|
+
return torch.stack([sample_mean, sample_logvar], dim=-1)
|
|
62
|
+
|
|
63
|
+
def sample(self, N):
|
|
64
|
+
with torch.no_grad():
|
|
65
|
+
return self.rsample(N)
|
|
66
|
+
|
|
67
|
+
def kl_div(self, target_dist):
|
|
68
|
+
kl1 = euclidean_kl_div(
|
|
69
|
+
self.normal_mu,
|
|
70
|
+
self.normal_logvar,
|
|
71
|
+
target_dist.normal_mu,
|
|
72
|
+
target_dist.normal_logvar
|
|
73
|
+
)
|
|
74
|
+
kl2 = gamma_kl_div(
|
|
75
|
+
self.gamma_a,
|
|
76
|
+
self.gamma_b,
|
|
77
|
+
target_dist.gamma_a,
|
|
78
|
+
target_dist.gamma_b
|
|
79
|
+
)
|
|
80
|
+
return kl1 + kl2
|
|
81
|
+
|