gnodevae 0.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
GNODEVAE/module.py ADDED
@@ -0,0 +1,939 @@
1
+ """
2
+ Graph Variational Autoencoder (GraphVAE) Modules
3
+
4
+ This module implements the core neural network architectures for GNODEVAE's
5
+ GraphVAE models. It provides flexible encoder and decoder components that can
6
+ be combined to create various VAE architectures for single-cell data analysis.
7
+
8
+ Key Components:
9
+ - GraphEncoder: Graph-based encoder using various GNN layers
10
+ - BilinearDecoder: Bilinear graph structure decoder
11
+ - InnerProductDecoder: Inner product graph structure decoder
12
+ - MLPDecoder: MLP-based graph structure decoder
13
+ - FeatureEncoder: Feature encoding network
14
+ - FeatureDecoder: Feature decoding network
15
+ - GraphVAE: Complete VAE model combining encoder and decoders
16
+ - iGraphVAE: Interpretable VAE with additional interpretable latent space
17
+ """
18
+
19
+ import torch
20
+ import torch.nn as nn
21
+ import torch.nn.functional as F
22
+ from torch_geometric.nn import (GCNConv,
23
+ ChebConv,
24
+ SAGEConv,
25
+ GraphConv,
26
+ TAGConv,
27
+ ARMAConv,
28
+ GATConv,
29
+ TransformerConv,
30
+ SGConv,
31
+ SSGConv)
32
+ from torch.distributions import Normal, kl_divergence
33
+ from typing import Optional, Tuple
34
+ from sklearn.neighbors import kneighbors_graph
35
+ import numpy as np
36
+
37
+ class GraphEncoder(nn.Module):
38
+ """
39
+ Flexible graph neural network encoder with variational output.
40
+
41
+ This encoder supports multiple types of graph convolution layers and produces
42
+ a probabilistic latent representation suitable for variational autoencoders.
43
+ It uses batch normalization and dropout for regularization, and optionally
44
+ includes residual connections for improved gradient flow.
45
+
46
+ The encoder architecture:
47
+ 1. Multiple graph convolutional layers with batch norm and dropout
48
+ 2. ReLU activation between layers
49
+ 3. Optional residual connection from first layer
50
+ 4. Two output layers for mean and log variance of latent distribution
51
+
52
+ Parameters
53
+ ----------
54
+ input_dim : int
55
+ Input feature dimension (number of genes after preprocessing).
56
+ hidden_dim : int
57
+ Hidden layer dimension for intermediate representations.
58
+ latent_dim : int
59
+ Latent space dimension for cell embeddings.
60
+ conv_layer_type : str, default='GAT'
61
+ Type of graph convolutional layer. Options:
62
+ - 'GCN': Graph Convolutional Network
63
+ - 'Cheb': Chebyshev spectral graph convolution
64
+ - 'SAGE': GraphSAGE
65
+ - 'Graph': Basic graph convolution
66
+ - 'TAG': Topology Adaptive Graph convolution
67
+ - 'ARMA': ARMA graph convolution
68
+ - 'GAT': Graph Attention Network (recommended)
69
+ - 'Transformer': Graph Transformer
70
+ - 'SG': Simplifying Graph convolution
71
+ - 'SSG': Simple Spectral Graph convolution
72
+ hidden_layers : int, default=2
73
+ Number of hidden layers (excluding input and output layers).
74
+ dropout : float, default=0.05
75
+ Dropout rate for regularization (applied after each layer).
76
+ Cheb_k : int, default=1
77
+ Order of Chebyshev polynomials for ChebConv layer.
78
+ alpha : float, default=0.5
79
+ Teleport probability for SSGConv layer.
80
+
81
+ Attributes
82
+ ----------
83
+ convs : nn.ModuleList
84
+ List of graph convolutional layers
85
+ bns : nn.ModuleList
86
+ List of batch normalization layers
87
+ dropouts : nn.ModuleList
88
+ List of dropout layers
89
+ conv_mean : nn.Module
90
+ Output layer for latent mean
91
+ conv_logvar : nn.Module
92
+ Output layer for latent log variance
93
+
94
+ Notes
95
+ -----
96
+ Different graph convolution types have different properties:
97
+ - GAT: Uses attention mechanism, good for heterogeneous graphs
98
+ - GCN: Classic choice, computationally efficient
99
+ - SAGE: Handles variable neighborhood sizes well
100
+ - Transformer: Most expressive but computationally expensive
101
+ """
102
+
103
+ def __init__(
104
+ self,
105
+ input_dim: int,
106
+ hidden_dim: int,
107
+ latent_dim: int,
108
+ conv_layer_type: str = 'GAT',
109
+ hidden_layers: int = 2,
110
+ dropout: float = 0.05,
111
+ Cheb_k: Optional[int] = 1,
112
+ alpha: Optional[float] = 0.5,
113
+ ):
114
+ super(GraphEncoder, self).__init__()
115
+
116
+ # Map string to the corresponding Torch Geometric layer
117
+ layer_map = {
118
+ 'GCN': GCNConv,
119
+ 'Cheb': ChebConv,
120
+ 'SAGE': SAGEConv,
121
+ 'Graph': GraphConv,
122
+ 'TAG': TAGConv,
123
+ 'ARMA': ARMAConv,
124
+ 'GAT': GATConv,
125
+ 'Transformer': TransformerConv,
126
+ 'SG': SGConv,
127
+ 'SSG': SSGConv
128
+ }
129
+ if conv_layer_type not in layer_map:
130
+ raise ValueError(f"Unsupported layer type: {conv_layer_type}. Choose from {list(layer_map.keys())}.")
131
+ self.conv_layer_type = conv_layer_type
132
+ self.conv_layer = layer_map[conv_layer_type]
133
+
134
+ self.hidden_layers = hidden_layers
135
+ self.dropout = dropout
136
+
137
+ # Define the layers dynamically
138
+ self.convs = nn.ModuleList()
139
+ self.bns = nn.ModuleList()
140
+ self.dropouts = nn.ModuleList()
141
+
142
+ # Input layer
143
+ if self.conv_layer_type == 'Transformer':
144
+ self.convs.append(self.conv_layer(input_dim, hidden_dim, edge_dim=1)) # Set edge_dim=1 for TransformerConv
145
+ elif self.conv_layer_type == 'Cheb':
146
+ self.convs.append(self.conv_layer(input_dim, hidden_dim, Cheb_k)) # Set k=Cheb_k for ChebConv
147
+ elif self.conv_layer_type == 'SSG':
148
+ self.convs.append(self.conv_layer(input_dim, hidden_dim, alpha=alpha)) # Set alpha for SSGConv
149
+ else:
150
+ self.convs.append(self.conv_layer(input_dim, hidden_dim))
151
+ self.bns.append(nn.BatchNorm1d(hidden_dim))
152
+ self.dropouts.append(nn.Dropout(dropout))
153
+
154
+ # Hidden layers
155
+ for _ in range(hidden_layers - 1):
156
+ if self.conv_layer_type == 'Transformer':
157
+ self.convs.append(self.conv_layer(hidden_dim, hidden_dim, edge_dim=1)) # Set edge_dim=1 for TransformerConv
158
+ elif self.conv_layer_type == 'Cheb':
159
+ self.convs.append(self.conv_layer(hidden_dim, hidden_dim, Cheb_k)) # Set k=Cheb_k for ChebConv
160
+ elif self.conv_layer_type == 'SSG':
161
+ self.convs.append(self.conv_layer(hidden_dim, hidden_dim, alpha=alpha)) # Set alpha for SSGConv
162
+ else:
163
+ self.convs.append(self.conv_layer(hidden_dim, hidden_dim))
164
+ self.bns.append(nn.BatchNorm1d(hidden_dim))
165
+ self.dropouts.append(nn.Dropout(dropout))
166
+
167
+ # Latent layers for mean and log variance
168
+ if self.conv_layer_type == 'Transformer':
169
+ self.conv_mean = self.conv_layer(hidden_dim, latent_dim, edge_dim=1) # Set edge_dim=1 for TransformerConv
170
+ self.conv_logvar = self.conv_layer(hidden_dim, latent_dim, edge_dim=1) # Set edge_dim=1 for TransformerConv
171
+ elif self.conv_layer_type == 'Cheb':
172
+ self.conv_mean = self.conv_layer(hidden_dim, latent_dim, Cheb_k) # Set k=Cheb_k for ChebConv
173
+ self.conv_logvar = self.conv_layer(hidden_dim, latent_dim, Cheb_k) # Set k=Cheb_k for ChebConv
174
+ elif self.conv_layer_type == 'SSG':
175
+ self.conv_mean = self.conv_layer(hidden_dim, latent_dim, alpha=alpha) # Set alpha for SSGConv
176
+ self.conv_logvar = self.conv_layer(hidden_dim, latent_dim, alpha=alpha) # Set alpha for SSGConv
177
+ else:
178
+ self.conv_mean = self.conv_layer(hidden_dim, latent_dim)
179
+ self.conv_logvar = self.conv_layer(hidden_dim, latent_dim)
180
+
181
+ self.relu = nn.ReLU()
182
+ self.apply(self.weight_init)
183
+
184
+ def weight_init(self, m: nn.Module) -> None:
185
+ """
186
+ Initialize weights for the layers.
187
+
188
+ Parameters
189
+ ----------
190
+ m : nn.Module
191
+ A module to initialize.
192
+ """
193
+ if isinstance(m, nn.Linear) or isinstance(m, nn.Conv1d):
194
+ nn.init.xavier_uniform_(m.weight)
195
+ if m.bias is not None:
196
+ nn.init.zeros_(m.bias)
197
+
198
+ def forward(
199
+ self,
200
+ x: torch.Tensor,
201
+ edge_index: torch.Tensor,
202
+ edge_weight: Optional[torch.Tensor] = None,
203
+ use_residual: bool = True,
204
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
205
+ """
206
+ Forward pass of the Graph Encoder.
207
+
208
+ Parameters
209
+ ----------
210
+ x : torch.Tensor
211
+ Node features, shape (num_nodes, input_dim).
212
+ edge_index : torch.Tensor
213
+ Edge indices, shape (2, num_edges).
214
+ edge_weight : torch.Tensor, optional
215
+ Edge weights, shape (num_edges,), by default None.
216
+ use_residual : bool, optional
217
+ Whether to use residual connections, by default True.
218
+
219
+ Returns
220
+ -------
221
+ q_z : torch.Tensor
222
+ Latent representation, shape (num_nodes, latent_dim).
223
+ q_m : torch.Tensor
224
+ Mean of the latent distribution, shape (num_nodes, latent_dim).
225
+ q_s : torch.Tensor
226
+ Standard deviation of the latent distribution, shape (num_nodes, latent_dim).
227
+ """
228
+ residual = None
229
+
230
+ # Apply convolutional layers
231
+ for i in range(self.hidden_layers):
232
+ # Handle different convolutional layers based on their type
233
+ if isinstance(self.convs[i], SAGEConv):
234
+ x = self.convs[i](x, edge_index) # Ignore edge_weight for SAGEConv
235
+ elif isinstance(self.convs[i], TransformerConv):
236
+ x = self.convs[i](x, edge_index, edge_weight.view(-1, 1)) # Reshape edge_weight for TransformerConv
237
+ else:
238
+ x = self.convs[i](x, edge_index, edge_weight) # Pass edge_weight for other layers
239
+
240
+ # Apply batch normalization, activation, and dropout
241
+ x = self.bns[i](x)
242
+ x = self.relu(x)
243
+ x = self.dropouts[i](x)
244
+
245
+ # Add residual connection for the first layer if enabled
246
+ if use_residual and i == 0:
247
+ residual = x
248
+
249
+ # Add residual connection after the last hidden layer if enabled
250
+ if use_residual and residual is not None:
251
+ x += residual
252
+
253
+ # Compute mean and log variance for the latent space
254
+ if isinstance(self.conv_mean, SAGEConv):
255
+ q_m = self.conv_mean(x, edge_index) # Ignore edge_weight for SAGEConv
256
+ elif isinstance(self.conv_mean, TransformerConv):
257
+ q_m = self.conv_mean(x, edge_index, edge_weight.view(-1, 1)) # Reshape edge_weight for TransformerConv
258
+ else:
259
+ q_m = self.conv_mean(x, edge_index, edge_weight) # Pass edge_weight for other layers
260
+
261
+ if isinstance(self.conv_logvar, SAGEConv):
262
+ q_s = self.conv_logvar(x, edge_index) # Ignore edge_weight for SAGEConv
263
+ elif isinstance(self.conv_logvar, TransformerConv):
264
+ q_s = self.conv_logvar(x, edge_index, edge_weight.view(-1, 1)) # Reshape edge_weight for TransformerConv
265
+ else:
266
+ q_s = self.conv_logvar(x, edge_index, edge_weight) # Pass edge_weight for other layers
267
+
268
+ # Compute standard deviation and sample from the latent distribution
269
+ s = F.softplus(q_s) + 1e-6
270
+ n = Normal(q_m, s)
271
+ q_z = n.rsample()
272
+
273
+ return q_z, q_m, q_s
274
+
275
+
276
+
277
+ class BilinearDecoder(nn.Module):
278
+ """
279
+ Bilinear graph structure decoder.
280
+
281
+ This decoder reconstructs the adjacency matrix using a learned bilinear
282
+ transformation. It computes edge probabilities as:
283
+ A_pred = sigmoid(Z * W * Z^T)
284
+ where Z is the latent embedding matrix and W is a learned weight matrix.
285
+
286
+ Parameters
287
+ ----------
288
+ latent_dim : int
289
+ Latent space dimension.
290
+
291
+ Attributes
292
+ ----------
293
+ weight : nn.Parameter
294
+ Learnable bilinear transformation matrix, shape (latent_dim, latent_dim).
295
+
296
+ Notes
297
+ -----
298
+ The bilinear decoder is more expressive than inner product but has
299
+ O(latent_dim^2) parameters, which can be memory-intensive for large
300
+ latent dimensions.
301
+ """
302
+
303
+ def __init__(self, latent_dim: int):
304
+ super(BilinearDecoder, self).__init__()
305
+ self.weight = nn.Parameter(torch.Tensor(latent_dim, latent_dim))
306
+ nn.init.xavier_uniform_(self.weight)
307
+
308
+ def forward(self, z: torch.Tensor) -> torch.Tensor:
309
+ """
310
+ Reconstruct adjacency matrix using bilinear transformation.
311
+
312
+ Parameters
313
+ ----------
314
+ z : torch.Tensor
315
+ Latent node embeddings, shape (num_nodes, latent_dim).
316
+
317
+ Returns
318
+ -------
319
+ adj_recon : torch.Tensor
320
+ Reconstructed adjacency matrix with edge probabilities,
321
+ shape (num_nodes, num_nodes). Values in [0, 1].
322
+ """
323
+ adj_recon = torch.sigmoid(torch.matmul(z @ self.weight, z.t()))
324
+ return adj_recon
325
+
326
+
327
+ class InnerProductDecoder(nn.Module):
328
+ """
329
+ Inner product graph structure decoder.
330
+
331
+ This is the simplest graph decoder that reconstructs edge probabilities
332
+ using the inner product of latent embeddings:
333
+ A_pred = sigmoid(Z * Z^T)
334
+
335
+ This decoder has no learnable parameters and is computationally efficient,
336
+ making it suitable for large graphs. It assumes that similar cells (high
337
+ inner product) should be connected.
338
+
339
+ Notes
340
+ -----
341
+ While parameter-free, this decoder is less expressive than bilinear or
342
+ MLP decoders. It works well when the latent space naturally encodes
343
+ cell similarity.
344
+ """
345
+
346
+ def __init__(self):
347
+ super(InnerProductDecoder, self).__init__()
348
+
349
+ def forward(self, z: torch.Tensor) -> torch.Tensor:
350
+ """
351
+ Reconstruct adjacency matrix using inner product.
352
+
353
+ Parameters
354
+ ----------
355
+ z : torch.Tensor
356
+ Latent node embeddings, shape (num_nodes, latent_dim).
357
+
358
+ Returns
359
+ -------
360
+ adj_recon : torch.Tensor
361
+ Reconstructed adjacency matrix with edge probabilities,
362
+ shape (num_nodes, num_nodes). Values in [0, 1].
363
+ """
364
+ adj_recon = torch.sigmoid(torch.matmul(z, z.t()))
365
+ return adj_recon
366
+
367
+
368
+ class MLPDecoder(nn.Module):
369
+ """
370
+ Multi-layer perceptron (MLP) graph structure decoder.
371
+
372
+ This decoder uses a neural network to predict edge probabilities from
373
+ concatenated node embeddings. It's the most expressive decoder type,
374
+ capable of learning complex edge probability functions.
375
+
376
+ Architecture: [latent_dim*2] -> [hidden_dim] -> ReLU -> [1] -> Sigmoid
377
+
378
+ Parameters
379
+ ----------
380
+ latent_dim : int
381
+ Latent space dimension.
382
+ hidden_dim : int
383
+ Hidden layer dimension for the MLP.
384
+
385
+ Attributes
386
+ ----------
387
+ mlp : nn.Sequential
388
+ MLP network for edge prediction.
389
+
390
+ Notes
391
+ -----
392
+ This decoder only computes edge probabilities for existing edges
393
+ (specified by edge_index), making it memory-efficient for sparse graphs.
394
+ For dense graphs, consider bilinear or inner product decoders.
395
+ """
396
+
397
+ def __init__(self, latent_dim: int, hidden_dim: int):
398
+ super(MLPDecoder, self).__init__()
399
+ self.mlp = nn.Sequential(
400
+ nn.Linear(latent_dim * 2, hidden_dim),
401
+ nn.ReLU(),
402
+ nn.Linear(hidden_dim, 1),
403
+ nn.Sigmoid(),
404
+ )
405
+
406
+ def forward(self, z: torch.Tensor, edge_index: torch.Tensor) -> torch.Tensor:
407
+ """
408
+ Reconstruct adjacency matrix using an MLP.
409
+
410
+ Parameters
411
+ ----------
412
+ z : torch.Tensor
413
+ Latent node embeddings, shape (num_nodes, latent_dim).
414
+ edge_index : torch.Tensor
415
+ Edge indices, shape (2, num_edges). Specifies which edges to predict.
416
+
417
+ Returns
418
+ -------
419
+ adj_recon : torch.Tensor
420
+ Reconstructed adjacency matrix with edge probabilities,
421
+ shape (num_nodes, num_nodes). Only predicted edges have non-zero values.
422
+ """
423
+ num_nodes = z.size(0)
424
+ row, col = edge_index
425
+ # Concatenate source and target node embeddings
426
+ edge_features = torch.cat([z[row], z[col]], dim=1)
427
+ edge_probs = self.mlp(edge_features).squeeze()
428
+ # Convert to adjacency matrix format
429
+ adj_recon = torch.zeros((num_nodes, num_nodes), device=z.device)
430
+ adj_recon[row, col] = edge_probs
431
+ return adj_recon
432
+
433
+
434
+ class FeatureEncoder(nn.Module):
435
+ """
436
+ Feature encoder for mapping high-dimensional data to latent space.
437
+
438
+ This encoder uses fully connected layers to encode features. Unlike
439
+ GraphEncoder, it doesn't use graph structure, making it suitable for
440
+ scenarios where graph information is unavailable or for auxiliary tasks.
441
+
442
+ Parameters
443
+ ----------
444
+ input_dim : int
445
+ Original input feature dimension (e.g., number of genes).
446
+ hidden_dim : int
447
+ Hidden layer dimension.
448
+ latent_dim : int
449
+ Latent space dimension (output dimension).
450
+ hidden_layers : int, default=2
451
+ Number of hidden layers in the encoder.
452
+
453
+ Attributes
454
+ ----------
455
+ nn : nn.Sequential
456
+ Sequential neural network for encoding.
457
+ disp : nn.Parameter
458
+ Dispersion parameter for negative binomial distribution (for gene expression modeling).
459
+ """
460
+
461
+ def __init__(self, input_dim: int, hidden_dim: int, latent_dim: int, hidden_layers: int = 2):
462
+ super(FeatureEncoder, self).__init__()
463
+ self.hidden_layers = hidden_layers
464
+
465
+ layers = []
466
+ layers.append(nn.Linear(input_dim, hidden_dim))
467
+ layers.append(nn.ReLU())
468
+
469
+ for _ in range(hidden_layers - 1):
470
+ layers.append(nn.Linear(hidden_dim, hidden_dim))
471
+ layers.append(nn.ReLU())
472
+
473
+ layers.append(nn.Linear(hidden_dim, latent_dim))
474
+
475
+ self.nn = nn.Sequential(*layers)
476
+ self.disp = nn.Parameter(torch.randn(input_dim))
477
+ self.apply(self.weight_init)
478
+
479
+ def weight_init(self, m: nn.Module) -> None:
480
+ """
481
+ Initialize weights using Xavier initialization.
482
+
483
+ Parameters
484
+ ----------
485
+ m : nn.Module
486
+ Module to initialize.
487
+ """
488
+ if isinstance(m, nn.Linear):
489
+ nn.init.xavier_uniform_(m.weight)
490
+ if m.bias is not None:
491
+ nn.init.zeros_(m.bias)
492
+
493
+ def forward(self, z: torch.Tensor) -> torch.Tensor:
494
+ """
495
+ Encode features to latent space.
496
+
497
+ Parameters
498
+ ----------
499
+ z : torch.Tensor
500
+ Input features, shape (num_cells, input_dim).
501
+
502
+ Returns
503
+ -------
504
+ torch.Tensor
505
+ Latent representation, shape (num_cells, latent_dim).
506
+ """
507
+ return self.nn(z)
508
+
509
+
510
+
511
+ class FeatureDecoder(nn.Module):
512
+ """
513
+ Feature decoder for reconstructing gene expression from latent space.
514
+
515
+ This decoder takes latent embeddings and reconstructs the original gene
516
+ expression profiles. It uses a softmax output to produce normalized
517
+ expression values, which is suitable for count-based data.
518
+
519
+ Parameters
520
+ ----------
521
+ input_dim : int
522
+ Original input feature dimension (number of genes to reconstruct).
523
+ hidden_dim : int
524
+ Hidden layer dimension.
525
+ latent_dim : int
526
+ Latent space dimension (input dimension).
527
+ hidden_layers : int, default=2
528
+ Number of hidden layers in the decoder.
529
+
530
+ Attributes
531
+ ----------
532
+ nn : nn.Sequential
533
+ Sequential neural network for decoding.
534
+ disp : nn.Parameter
535
+ Dispersion parameter for modeling gene expression variance.
536
+
537
+ Notes
538
+ -----
539
+ The softmax output ensures that reconstructed values sum to a constant,
540
+ which is appropriate for compositional data like gene expression after
541
+ normalization. For count data, consider using a negative binomial
542
+ likelihood in the loss function.
543
+ """
544
+
545
+ def __init__(self, input_dim: int, hidden_dim: int, latent_dim: int, hidden_layers: int = 2):
546
+ super(FeatureDecoder, self).__init__()
547
+ self.hidden_layers = hidden_layers
548
+
549
+ layers = []
550
+ layers.append(nn.Linear(latent_dim, hidden_dim))
551
+ layers.append(nn.ReLU())
552
+
553
+ for _ in range(hidden_layers - 1):
554
+ layers.append(nn.Linear(hidden_dim, hidden_dim))
555
+ layers.append(nn.ReLU())
556
+
557
+ layers.append(nn.Linear(hidden_dim, input_dim))
558
+ layers.append(nn.Softmax(dim=-1)) # Convert to probability distribution
559
+
560
+ self.nn = nn.Sequential(*layers)
561
+ self.disp = nn.Parameter(torch.randn(input_dim))
562
+ self.apply(self.weight_init)
563
+
564
+ def weight_init(self, m: nn.Module) -> None:
565
+ """
566
+ Initialize weights using Xavier initialization.
567
+
568
+ Parameters
569
+ ----------
570
+ m : nn.Module
571
+ Module to initialize.
572
+ """
573
+ if isinstance(m, nn.Linear):
574
+ nn.init.xavier_uniform_(m.weight)
575
+ if m.bias is not None:
576
+ nn.init.zeros_(m.bias)
577
+
578
+ def forward(self, z: torch.Tensor) -> torch.Tensor:
579
+ """
580
+ Reconstruct gene expression from latent embeddings.
581
+
582
+ Parameters
583
+ ----------
584
+ z : torch.Tensor
585
+ Latent node embeddings, shape (num_cells, latent_dim).
586
+
587
+ Returns
588
+ -------
589
+ recon_features : torch.Tensor
590
+ Reconstructed gene expression, shape (num_cells, input_dim).
591
+ Values are normalized via softmax.
592
+ """
593
+ return self.nn(z)
594
+
595
+
596
+ class GraphVAE(nn.Module):
597
+ """
598
+ Complete Graph Variational Autoencoder model.
599
+
600
+ This class combines a graph encoder, structure decoder, and feature decoder
601
+ to create a complete VAE model for single-cell data. The model learns to:
602
+ 1. Encode cells into a low-dimensional latent space using graph structure
603
+ 2. Reconstruct the cell-cell similarity graph
604
+ 3. Reconstruct gene expression profiles
605
+
606
+ The VAE framework enables:
607
+ - Learning of robust latent representations
608
+ - Handling of noise and batch effects
609
+ - Uncertainty quantification via the probabilistic latent space
610
+
611
+ Parameters
612
+ ----------
613
+ input_dim : int
614
+ Input feature dimension (number of genes).
615
+ hidden_dim : int
616
+ Hidden layer dimension for encoder and decoder.
617
+ latent_dim : int
618
+ Latent space dimension for cell embeddings.
619
+ encoder_type : str, default='GAT'
620
+ Type of graph convolutional layer for the encoder.
621
+ Options: 'GCN', 'Cheb', 'SAGE', 'TAG', 'ARMA', 'GAT', 'Transformer'.
622
+ encoder_hidden_layers : int, default=2
623
+ Number of hidden layers in the graph encoder.
624
+ decoder_type : str, default='MLP'
625
+ Type of graph decoder for structure reconstruction.
626
+ Options: 'Bilinear', 'InnerProduct', 'MLP'.
627
+ decoder_hidden_dim : int, default=128
628
+ Hidden dimension for the MLP decoder (if used).
629
+ feature_decoder_hidden_layers : int, default=2
630
+ Number of hidden layers in the feature decoder.
631
+ dropout : float, default=0.05
632
+ Dropout rate for regularization.
633
+ use_residual : bool, default=True
634
+ Whether to use residual connections in the encoder.
635
+ Cheb_k : int, default=1
636
+ Order of Chebyshev polynomials (for ChebConv).
637
+ alpha : float, default=0.5
638
+ Teleport probability (for SSGConv).
639
+
640
+ Attributes
641
+ ----------
642
+ g_encoder : GraphEncoder
643
+ Graph-based encoder network.
644
+ a_decoder : BilinearDecoder, InnerProductDecoder, or MLPDecoder
645
+ Decoder for reconstructing graph structure.
646
+ x_decoder : FeatureDecoder
647
+ Decoder for reconstructing gene expression.
648
+ use_residual : bool
649
+ Whether residual connections are enabled.
650
+
651
+ Examples
652
+ --------
653
+ >>> model = GraphVAE(
654
+ ... input_dim=2000,
655
+ ... hidden_dim=128,
656
+ ... latent_dim=10,
657
+ ... encoder_type='GAT'
658
+ ... )
659
+ >>> q_z, q_m, q_s, pred_a, pred_x = model(x, edge_index, edge_weight)
660
+
661
+ Notes
662
+ -----
663
+ The model uses the reparameterization trick for backpropagation through
664
+ the stochastic latent variables. The KL divergence between the learned
665
+ latent distribution and a standard normal prior is used as a regularizer.
666
+ """
667
+
668
+ def __init__(
669
+ self,
670
+ input_dim: int,
671
+ hidden_dim: int,
672
+ latent_dim: int,
673
+ encoder_type: str = 'GAT',
674
+ encoder_hidden_layers: int = 2,
675
+ decoder_type: str = 'MLP',
676
+ decoder_hidden_dim: int = 128,
677
+ feature_decoder_hidden_layers: int = 2,
678
+ dropout: float = 0.05,
679
+ use_residual: bool = True,
680
+ Cheb_k: Optional[int] = 1,
681
+ alpha: Optional[float] = .5
682
+ ):
683
+ super(GraphVAE, self).__init__()
684
+
685
+ # Initialize the graph encoder
686
+ self.g_encoder = GraphEncoder(
687
+ input_dim=input_dim,
688
+ hidden_dim=hidden_dim,
689
+ latent_dim=latent_dim,
690
+ conv_layer_type=encoder_type,
691
+ hidden_layers=encoder_hidden_layers,
692
+ dropout=dropout,
693
+ Cheb_k=Cheb_k,
694
+ alpha=alpha
695
+ )
696
+
697
+ # Initialize the graph structure decoder based on type
698
+ if decoder_type == 'Bilinear':
699
+ self.a_decoder = BilinearDecoder(latent_dim)
700
+ elif decoder_type == 'InnerProduct':
701
+ self.a_decoder = InnerProductDecoder()
702
+ elif decoder_type == 'MLP':
703
+ if decoder_hidden_dim is None:
704
+ raise ValueError("`decoder_hidden_dim` must be specified for MLPDecoder.")
705
+ self.a_decoder = MLPDecoder(latent_dim, decoder_hidden_dim)
706
+ else:
707
+ raise ValueError(f"Unsupported decoder type: {decoder_type}. Choose from ['Bilinear', 'InnerProduct', 'MLP'].")
708
+
709
+ # Initialize the feature decoder
710
+ self.x_decoder = FeatureDecoder(
711
+ input_dim=input_dim,
712
+ hidden_dim=hidden_dim,
713
+ latent_dim=latent_dim,
714
+ hidden_layers=feature_decoder_hidden_layers,
715
+ )
716
+ self.use_residual = use_residual
717
+
718
+ def forward(
719
+ self,
720
+ x: torch.Tensor,
721
+ edge_index: torch.Tensor,
722
+ edge_weight: Optional[torch.Tensor] = None
723
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
724
+ """
725
+ Forward pass through the GraphVAE model.
726
+
727
+ Parameters
728
+ ----------
729
+ x : torch.Tensor
730
+ Node features (gene expression), shape (num_cells, input_dim).
731
+ edge_index : torch.Tensor
732
+ Edge indices for cell-cell graph, shape (2, num_edges).
733
+ edge_weight : torch.Tensor, optional
734
+ Edge weights, shape (num_edges,).
735
+
736
+ Returns
737
+ -------
738
+ q_z : torch.Tensor
739
+ Sampled latent representation, shape (num_cells, latent_dim).
740
+ q_m : torch.Tensor
741
+ Mean of the latent distribution, shape (num_cells, latent_dim).
742
+ q_s : torch.Tensor
743
+ Log variance of the latent distribution, shape (num_cells, latent_dim).
744
+ pred_a : torch.Tensor
745
+ Reconstructed adjacency matrix or edge probabilities,
746
+ shape (num_cells, num_cells).
747
+ pred_x : torch.Tensor
748
+ Reconstructed gene expression, shape (num_cells, input_dim).
749
+
750
+ Notes
751
+ -----
752
+ The latent variables are sampled using the reparameterization trick:
753
+ z = μ + σ * ε, where ε ~ N(0, 1)
754
+ This enables backpropagation through the sampling operation.
755
+ """
756
+ # Step 1: Encode to latent space
757
+ q_z, q_m, q_s = self.g_encoder(x, edge_index, edge_weight, self.use_residual)
758
+
759
+ # Step 2: Decode graph structure
760
+ if isinstance(self.a_decoder, MLPDecoder):
761
+ # MLPDecoder requires edge_index to know which edges to predict
762
+ pred_a = self.a_decoder(q_z, edge_index)
763
+ else:
764
+ # Bilinear and InnerProduct decoders predict all possible edges
765
+ pred_a = self.a_decoder(q_z)
766
+
767
+ # Step 3: Decode gene expression
768
+ pred_x = self.x_decoder(q_z)
769
+
770
+ return q_z, q_m, q_s, pred_a, pred_x
771
+
772
+
773
+ class iGraphVAE(nn.Module):
774
+ """
775
+ Interpretable Variational Graph Autoencoder.
776
+
777
+ Parameters
778
+ ----------
779
+ input_dim : int
780
+ Input feature dimension.
781
+ hidden_dim : int
782
+ Hidden layer dimension for the encoder.
783
+ latent_dim : int
784
+ Latent space dimension.
785
+ idim : int
786
+ Interpretable latent space dimension.
787
+ encoder_type : str, optional
788
+ Type of graph convolutional layer, e.g., 'GCN', 'Cheb', 'SAGE', etc.
789
+ encoder_hidden_layers : int, optional
790
+ Number of hidden layers in the graph encoder.
791
+ decoder_type : str, optional
792
+ Type of graph decoder, e.g., 'Bilinear', 'InnerProduct', 'MLP'.
793
+ decoder_hidden_dim : int, optional
794
+ Hidden dimension for the MLPDecoder (if used).
795
+ feature_decoder_hidden_layers : int, optional
796
+ Number of hidden layers in the feature decoder.
797
+ dropout : float, optional
798
+ Dropout rate.
799
+ use_residual : bool, optional
800
+ Whether to use residual connections.
801
+ Cheb_k : int, optional
802
+ The order of Chebyshev polynomials for ChebConv.
803
+ alpha : float, optional
804
+ Teleport probability.
805
+ """
806
+
807
+ def __init__(
808
+ self,
809
+ input_dim: int,
810
+ hidden_dim: int,
811
+ latent_dim: int,
812
+ idim: int,
813
+ encoder_type: str = 'GAT',
814
+ encoder_hidden_layers: int = 2,
815
+ decoder_type: str = 'MLP',
816
+ decoder_hidden_dim: int = 128,
817
+ feature_decoder_hidden_layers: int = 2,
818
+ dropout: float = 0.05,
819
+ use_residual: bool = True,
820
+ Cheb_k: int = 1,
821
+ alpha: float = 0.5
822
+ ):
823
+ super(iGraphVAE, self).__init__()
824
+
825
+ # Initialize the primary graph encoder
826
+ self.g_encoder = GraphEncoder(
827
+ input_dim=input_dim,
828
+ hidden_dim=hidden_dim,
829
+ latent_dim=latent_dim,
830
+ conv_layer_type=encoder_type,
831
+ hidden_layers=encoder_hidden_layers,
832
+ dropout=dropout,
833
+ Cheb_k=Cheb_k,
834
+ alpha=alpha
835
+ )
836
+
837
+ # Initialize the interpretable graph encoder
838
+ self.ig_encoder = FeatureEncoder(
839
+ input_dim=latent_dim,
840
+ hidden_dim=hidden_dim,
841
+ latent_dim=idim,
842
+ hidden_layers=encoder_hidden_layers
843
+ )
844
+
845
+ # Initialize the graph decoders
846
+ if decoder_type == 'Bilinear':
847
+ self.a_decoder = BilinearDecoder(latent_dim)
848
+ elif decoder_type == 'InnerProduct':
849
+ self.a_decoder = InnerProductDecoder()
850
+ elif decoder_type == 'MLP':
851
+ if decoder_hidden_dim is None:
852
+ raise ValueError("`decoder_hidden_dim` must be specified for MLPDecoder.")
853
+ self.a_decoder = MLPDecoder(latent_dim, decoder_hidden_dim)
854
+ else:
855
+ raise ValueError(f"Unsupported decoder type: {decoder_type}. Choose from ['Bilinear', 'InnerProduct', 'MLP'].")
856
+
857
+ # Initialize the feature decoders
858
+ self.x_decoder = FeatureDecoder(
859
+ input_dim=input_dim,
860
+ hidden_dim=hidden_dim,
861
+ latent_dim=latent_dim,
862
+ hidden_layers=feature_decoder_hidden_layers,
863
+ )
864
+ self.ix_decoder = FeatureDecoder(
865
+ input_dim=latent_dim,
866
+ hidden_dim=hidden_dim,
867
+ latent_dim=idim,
868
+ hidden_layers=feature_decoder_hidden_layers,
869
+ )
870
+
871
+ self.use_residual = use_residual
872
+
873
+ def forward(
874
+ self,
875
+ x: torch.Tensor,
876
+ edge_index: torch.Tensor,
877
+ edge_weight: torch.Tensor = None
878
+ ) -> tuple:
879
+ """
880
+ Forward pass of the Interpretable GraphVAE.
881
+
882
+ Parameters
883
+ ----------
884
+ x : torch.Tensor
885
+ Node features, shape (num_nodes, input_dim).
886
+ edge_index : torch.Tensor
887
+ Edge indices, shape (2, num_edges).
888
+ edge_weight : torch.Tensor, optional
889
+ Edge weights, shape (num_edges,).
890
+
891
+ Returns
892
+ -------
893
+ q_z : torch.Tensor
894
+ Latent representation, shape (num_nodes, latent_dim).
895
+ q_m : torch.Tensor
896
+ Mean of the latent distribution, shape (num_nodes, latent_dim).
897
+ q_s : torch.Tensor
898
+ Standard deviation of the latent distribution, shape (num_nodes, latent_dim).
899
+ pred_a : torch.Tensor
900
+ Reconstructed adjacency matrix or edge probabilities.
901
+ pred_x : torch.Tensor
902
+ Reconstructed node features, shape (num_nodes, input_dim).
903
+ iloss : torch.Tensor
904
+ Loss associated with the interpretable components.
905
+ """
906
+ # Encode the graph to obtain primary latent variables
907
+ q_z, q_m, q_s = self.g_encoder(x, edge_index, edge_weight, self.use_residual)
908
+ device = q_z.device
909
+
910
+ # Interpretable encoding
911
+ # Create k-nearest neighbor graph from latent representation
912
+ ix = q_z.detach().cpu().numpy()
913
+ knn_graph = kneighbors_graph(ix, n_neighbors=5, mode='connectivity', include_self=False)
914
+ iedge_index_np, iedge_weight_np = knn_graph.nonzero()
915
+ iedge_index = torch.tensor(np.array([iedge_index_np, iedge_weight_np]), dtype=torch.long).to(device)
916
+ iedge_weight = torch.tensor(knn_graph.data, dtype=torch.float).to(device)
917
+
918
+ # Interpretable graph encoding
919
+ i_q_z = self.ig_encoder(q_z)
920
+
921
+ # Decode the node features
922
+ pred_x = self.x_decoder(q_z)
923
+
924
+ pred_q_z = self.ix_decoder(i_q_z)
925
+
926
+ pred_ix = self.x_decoder(pred_q_z)
927
+
928
+ # Decode the primary graph structure
929
+ if isinstance(self.a_decoder, MLPDecoder):
930
+ # MLPDecoder requires edge_index
931
+ pred_a = self.a_decoder(q_z, edge_index)
932
+ pred_ia = self.a_decoder(pred_q_z, edge_index)
933
+ else:
934
+ # Other decoders do not require edge_index
935
+ pred_a = self.a_decoder(q_z)
936
+ pred_ia = self.a_decoder(pred_q_z)
937
+
938
+ return q_z, q_m, q_s, pred_a, pred_x, i_q_z, pred_ia, pred_ix
939
+