gnodevae 0.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- GNODEVAE/CLEAR.py +172 -0
- GNODEVAE/GODEVAE.py +413 -0
- GNODEVAE/GODEVAE_agent.py +385 -0
- GNODEVAE/GODEVAE_env.py +758 -0
- GNODEVAE/GODEVAE_trainer.py +309 -0
- GNODEVAE/SC3.py +198 -0
- GNODEVAE/__init__.py +6 -0
- GNODEVAE/agent.py +472 -0
- GNODEVAE/env.py +474 -0
- GNODEVAE/mixin.py +418 -0
- GNODEVAE/module.py +939 -0
- GNODEVAE/module_refined.py +561 -0
- GNODEVAE/partition_time.py +185 -0
- GNODEVAE/scGCC.py +184 -0
- GNODEVAE/scGNN.py +146 -0
- GNODEVAE/trainer.py +397 -0
- GNODEVAE/utils.py +460 -0
- GNODEVAE/utils_ODE.py +140 -0
- gnodevae-0.0.3.dist-info/METADATA +330 -0
- gnodevae-0.0.3.dist-info/RECORD +23 -0
- gnodevae-0.0.3.dist-info/WHEEL +5 -0
- gnodevae-0.0.3.dist-info/licenses/LICENSE +21 -0
- gnodevae-0.0.3.dist-info/top_level.txt +1 -0
GNODEVAE/GODEVAE_env.py
ADDED
|
@@ -0,0 +1,758 @@
|
|
|
1
|
+
from .GODEVAE_trainer import GODEVAE_Trainer_r
|
|
2
|
+
from .mixin import envMixin, scMixin
|
|
3
|
+
import numpy as np
|
|
4
|
+
import torch
|
|
5
|
+
from torch.utils.data import Dataset
|
|
6
|
+
from torch_geometric.loader import ClusterData
|
|
7
|
+
from torch_geometric.data import Data, DataLoader
|
|
8
|
+
from sklearn.cluster import KMeans
|
|
9
|
+
from typing import Optional, List, Tuple
|
|
10
|
+
from anndata import AnnData
|
|
11
|
+
import scanpy as sc
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class SubgraphDataset(Dataset):
|
|
16
|
+
"""
|
|
17
|
+
用于子图采样的数据集类 - 返回torch_geometric.data.Data对象
|
|
18
|
+
"""
|
|
19
|
+
def __init__(self,
|
|
20
|
+
node_features: np.ndarray,
|
|
21
|
+
edge_index: np.ndarray,
|
|
22
|
+
edge_weight: np.ndarray,
|
|
23
|
+
node_labels: np.ndarray,
|
|
24
|
+
device: torch.device,
|
|
25
|
+
subgraph_size: int = 512):
|
|
26
|
+
"""
|
|
27
|
+
Parameters
|
|
28
|
+
----------
|
|
29
|
+
node_features : np.ndarray
|
|
30
|
+
节点特征矩阵
|
|
31
|
+
edge_index : np.ndarray
|
|
32
|
+
边索引 (2, num_edges)
|
|
33
|
+
edge_weight : np.ndarray
|
|
34
|
+
边权重
|
|
35
|
+
node_labels : np.ndarray
|
|
36
|
+
节点标签(y值,通常是节点索引)
|
|
37
|
+
device : torch.device
|
|
38
|
+
设备
|
|
39
|
+
subgraph_size : int
|
|
40
|
+
子图大小(每个batch的节点数)
|
|
41
|
+
"""
|
|
42
|
+
self.node_features = node_features
|
|
43
|
+
self.edge_index = edge_index
|
|
44
|
+
self.edge_weight = edge_weight
|
|
45
|
+
self.node_labels = node_labels # 这应该是self.y
|
|
46
|
+
self.device = device
|
|
47
|
+
self.subgraph_size = subgraph_size
|
|
48
|
+
self.num_nodes = node_features.shape[0]
|
|
49
|
+
|
|
50
|
+
# 预计算每个节点的邻居(用于采样)
|
|
51
|
+
self.neighbors = self._compute_neighbors()
|
|
52
|
+
|
|
53
|
+
def _compute_neighbors(self):
|
|
54
|
+
"""计算每个节点的邻居"""
|
|
55
|
+
neighbors = [[] for _ in range(self.num_nodes)]
|
|
56
|
+
for i, j in self.edge_index.T:
|
|
57
|
+
neighbors[i].append(j)
|
|
58
|
+
if i != j: # 避免重复添加自环
|
|
59
|
+
neighbors[j].append(i)
|
|
60
|
+
return neighbors
|
|
61
|
+
|
|
62
|
+
def __len__(self):
|
|
63
|
+
# 返回可以采样的子图数量
|
|
64
|
+
return max(1, self.num_nodes // self.subgraph_size * 2) # 增加采样次数
|
|
65
|
+
|
|
66
|
+
def __getitem__(self, idx):
|
|
67
|
+
"""
|
|
68
|
+
采样子图并返回torch_geometric.data.Data对象
|
|
69
|
+
"""
|
|
70
|
+
# 随机采样节点
|
|
71
|
+
selected_nodes = self._random_node_sampling()
|
|
72
|
+
|
|
73
|
+
# 创建torch_geometric.data.Data对象
|
|
74
|
+
subgraph_data = self._create_data_object(selected_nodes)
|
|
75
|
+
|
|
76
|
+
return subgraph_data
|
|
77
|
+
|
|
78
|
+
def _random_node_sampling(self):
|
|
79
|
+
"""随机采样节点"""
|
|
80
|
+
num_sample = min(self.subgraph_size, self.num_nodes)
|
|
81
|
+
selected_nodes = np.random.choice(
|
|
82
|
+
self.num_nodes,
|
|
83
|
+
size=num_sample,
|
|
84
|
+
replace=False
|
|
85
|
+
)
|
|
86
|
+
return selected_nodes
|
|
87
|
+
|
|
88
|
+
def _neighbor_sampling(self, num_seeds=None):
|
|
89
|
+
"""基于邻居的采样"""
|
|
90
|
+
if num_seeds is None:
|
|
91
|
+
num_seeds = min(self.subgraph_size // 4, self.num_nodes)
|
|
92
|
+
|
|
93
|
+
# 选择种子节点
|
|
94
|
+
seed_nodes = np.random.choice(
|
|
95
|
+
self.num_nodes,
|
|
96
|
+
size=num_seeds,
|
|
97
|
+
replace=False
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
selected_nodes = set(seed_nodes)
|
|
101
|
+
|
|
102
|
+
# 扩展到邻居节点
|
|
103
|
+
for seed in seed_nodes:
|
|
104
|
+
neighbors = self.neighbors[seed]
|
|
105
|
+
if neighbors and len(selected_nodes) < self.subgraph_size:
|
|
106
|
+
# 计算还需要多少邻居
|
|
107
|
+
remaining_slots = self.subgraph_size - len(selected_nodes)
|
|
108
|
+
remaining_seeds = len([s for s in seed_nodes if s not in selected_nodes or s == seed])
|
|
109
|
+
|
|
110
|
+
if remaining_seeds > 0:
|
|
111
|
+
num_neighbors = min(
|
|
112
|
+
len(neighbors),
|
|
113
|
+
max(1, remaining_slots // remaining_seeds)
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
if num_neighbors > 0:
|
|
117
|
+
selected_neighbors = np.random.choice(
|
|
118
|
+
neighbors,
|
|
119
|
+
size=num_neighbors,
|
|
120
|
+
replace=False
|
|
121
|
+
)
|
|
122
|
+
selected_nodes.update(selected_neighbors)
|
|
123
|
+
|
|
124
|
+
return np.array(list(selected_nodes))
|
|
125
|
+
|
|
126
|
+
def _create_data_object(self, selected_nodes):
|
|
127
|
+
"""
|
|
128
|
+
创建torch_geometric.data.Data对象
|
|
129
|
+
这个对象必须与GODEVAE_Trainer_r期望的格式一致
|
|
130
|
+
"""
|
|
131
|
+
# 创建节点映射:原始索引 -> 新索引
|
|
132
|
+
node_map = {old_idx: new_idx for new_idx, old_idx in enumerate(selected_nodes)}
|
|
133
|
+
|
|
134
|
+
# 提取子图的边
|
|
135
|
+
edge_mask = np.isin(self.edge_index[0], selected_nodes) & np.isin(self.edge_index[1], selected_nodes)
|
|
136
|
+
subgraph_edges = self.edge_index[:, edge_mask]
|
|
137
|
+
subgraph_weights = self.edge_weight[edge_mask]
|
|
138
|
+
|
|
139
|
+
# 重新映射边索引到新的节点索引
|
|
140
|
+
new_edge_index = np.array([
|
|
141
|
+
[node_map[i] for i in subgraph_edges[0]],
|
|
142
|
+
[node_map[i] for i in subgraph_edges[1]]
|
|
143
|
+
])
|
|
144
|
+
|
|
145
|
+
# 提取节点特征
|
|
146
|
+
subgraph_features = self.node_features[selected_nodes]
|
|
147
|
+
|
|
148
|
+
# 提取节点标签(重新映射到子图的节点索引)
|
|
149
|
+
subgraph_y = np.array([node_map[original_idx] for original_idx in selected_nodes])
|
|
150
|
+
|
|
151
|
+
# 创建Data对象 - 确保属性名与原始代码一致
|
|
152
|
+
data = Data(
|
|
153
|
+
x=torch.tensor(subgraph_features, dtype=torch.float, device=self.device),
|
|
154
|
+
edge_index=torch.tensor(new_edge_index, dtype=torch.long, device=self.device),
|
|
155
|
+
edge_attr=torch.tensor(subgraph_weights, dtype=torch.float, device=self.device),
|
|
156
|
+
y=torch.tensor(subgraph_y, dtype=torch.long, device=self.device)
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
# 添加原始节点索引信息(用于后续重构)
|
|
160
|
+
data.original_node_idx = torch.tensor(selected_nodes, dtype=torch.long, device=self.device)
|
|
161
|
+
|
|
162
|
+
return data
|
|
163
|
+
|
|
164
|
+
class GNODEVAE_Env_Subgraph(GODEVAE_Trainer_r, envMixin, scMixin):
|
|
165
|
+
"""
|
|
166
|
+
使用子图采样的环境类 - 与原始API完全一致
|
|
167
|
+
"""
|
|
168
|
+
|
|
169
|
+
def __init__(self,
|
|
170
|
+
adata: AnnData,
|
|
171
|
+
layer: str,
|
|
172
|
+
n_var: int,
|
|
173
|
+
tech: str,
|
|
174
|
+
n_neighbors: int,
|
|
175
|
+
batch_tech: Optional[str],
|
|
176
|
+
all_feat: bool,
|
|
177
|
+
hidden_dim: int,
|
|
178
|
+
latent_dim: int,
|
|
179
|
+
ode_hidden_dim: int,
|
|
180
|
+
encoder_type: str,
|
|
181
|
+
graph_type: str,
|
|
182
|
+
structure_decoder_type: str,
|
|
183
|
+
feature_decoder_type: str,
|
|
184
|
+
hidden_layers: int,
|
|
185
|
+
decoder_hidden_dim: int,
|
|
186
|
+
dropout: float,
|
|
187
|
+
use_residual: bool,
|
|
188
|
+
Cheb_k: int,
|
|
189
|
+
alpha: float,
|
|
190
|
+
threshold: float,
|
|
191
|
+
sparse_threshold: Optional[int],
|
|
192
|
+
lr: float,
|
|
193
|
+
beta: float,
|
|
194
|
+
graph: float,
|
|
195
|
+
w_recon: float,
|
|
196
|
+
w_kl: float,
|
|
197
|
+
w_adj: float,
|
|
198
|
+
w_recon_ode: float,
|
|
199
|
+
w_z_div: float,
|
|
200
|
+
device: torch.device,
|
|
201
|
+
latent_type: str,
|
|
202
|
+
# 子图采样特有参数
|
|
203
|
+
subgraph_size: int,
|
|
204
|
+
num_subgraphs_per_epoch: int,
|
|
205
|
+
sampling_method: str, # 'random' or 'neighbor'
|
|
206
|
+
*args,
|
|
207
|
+
**kwargs):
|
|
208
|
+
|
|
209
|
+
# 首先注册AnnData并处理数据
|
|
210
|
+
self._register_adata(adata, layer, n_var, tech, n_neighbors, latent_dim, batch_tech, all_feat)
|
|
211
|
+
|
|
212
|
+
# 初始化父类GODEVAE_Trainer_r
|
|
213
|
+
super().__init__(
|
|
214
|
+
self.n_var,
|
|
215
|
+
hidden_dim,
|
|
216
|
+
latent_dim,
|
|
217
|
+
ode_hidden_dim,
|
|
218
|
+
encoder_type,
|
|
219
|
+
graph_type,
|
|
220
|
+
structure_decoder_type,
|
|
221
|
+
feature_decoder_type,
|
|
222
|
+
hidden_layers,
|
|
223
|
+
decoder_hidden_dim,
|
|
224
|
+
dropout,
|
|
225
|
+
use_residual,
|
|
226
|
+
Cheb_k,
|
|
227
|
+
alpha,
|
|
228
|
+
threshold,
|
|
229
|
+
sparse_threshold,
|
|
230
|
+
lr,
|
|
231
|
+
beta,
|
|
232
|
+
graph,
|
|
233
|
+
w_recon,
|
|
234
|
+
w_kl,
|
|
235
|
+
w_adj,
|
|
236
|
+
w_recon_ode,
|
|
237
|
+
w_z_div,
|
|
238
|
+
device,
|
|
239
|
+
latent_type,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
# 注册子图采样数据
|
|
243
|
+
self._register_subgraph_data(subgraph_size, num_subgraphs_per_epoch, sampling_method)
|
|
244
|
+
|
|
245
|
+
# 初始化评分列表
|
|
246
|
+
self.score: List[Tuple[float, float, float, float, float, float]] = []
|
|
247
|
+
self.mix_score: List[Tuple[float, float, float, float, float, float]] = []
|
|
248
|
+
|
|
249
|
+
def _register_adata(self,
|
|
250
|
+
adata: AnnData,
|
|
251
|
+
layer: str,
|
|
252
|
+
n_var: int,
|
|
253
|
+
tech: str,
|
|
254
|
+
n_neighbors: int,
|
|
255
|
+
latent_dim: int,
|
|
256
|
+
batch_tech: Optional[str],
|
|
257
|
+
all_feat: bool) -> None:
|
|
258
|
+
"""
|
|
259
|
+
注册AnnData对象并预处理数据 - 与原版本完全一致
|
|
260
|
+
"""
|
|
261
|
+
self._preprocess(adata, layer, n_var)
|
|
262
|
+
self._decomposition(adata, tech, latent_dim)
|
|
263
|
+
|
|
264
|
+
if batch_tech:
|
|
265
|
+
self._batchcorrect(adata, batch_tech, tech, layer)
|
|
266
|
+
|
|
267
|
+
# 确定用于邻域图的表示
|
|
268
|
+
if batch_tech == 'harmony':
|
|
269
|
+
use_rep = f'X_harmony_{tech}'
|
|
270
|
+
elif batch_tech == 'scvi':
|
|
271
|
+
use_rep = 'X_scvi'
|
|
272
|
+
else:
|
|
273
|
+
use_rep = f'X_{tech}'
|
|
274
|
+
|
|
275
|
+
# 构建邻域图
|
|
276
|
+
sc.pp.neighbors(adata, n_neighbors=n_neighbors, use_rep=use_rep)
|
|
277
|
+
|
|
278
|
+
# 选择特征
|
|
279
|
+
if all_feat:
|
|
280
|
+
self.X = np.log1p(adata.layers[layer].toarray())
|
|
281
|
+
else:
|
|
282
|
+
self.X = adata[:, adata.var['highly_variable']].X.toarray()
|
|
283
|
+
|
|
284
|
+
self.n_obs, self.n_var = self.X.shape
|
|
285
|
+
|
|
286
|
+
# 执行聚类
|
|
287
|
+
self.labels = KMeans(n_clusters=latent_dim).fit_predict(self.X)
|
|
288
|
+
|
|
289
|
+
# 提取图信息
|
|
290
|
+
coo = adata.obsp['connectivities'].tocoo()
|
|
291
|
+
self.edge_index = np.array([coo.row, coo.col])
|
|
292
|
+
self.edge_weight = coo.data
|
|
293
|
+
|
|
294
|
+
# 分配索引 - 这个y就是节点索引
|
|
295
|
+
self.y = np.arange(adata.shape[0])
|
|
296
|
+
self.idx = np.arange(adata.shape[0])
|
|
297
|
+
|
|
298
|
+
def _register_subgraph_data(self,
|
|
299
|
+
subgraph_size: int,
|
|
300
|
+
num_subgraphs_per_epoch: int,
|
|
301
|
+
sampling_method: str):
|
|
302
|
+
"""
|
|
303
|
+
注册子图采样数据
|
|
304
|
+
"""
|
|
305
|
+
self.subgraph_size = subgraph_size
|
|
306
|
+
self.num_subgraphs_per_epoch = num_subgraphs_per_epoch
|
|
307
|
+
self.sampling_method = sampling_method
|
|
308
|
+
|
|
309
|
+
# 创建子图数据集
|
|
310
|
+
self.subgraph_dataset = SubgraphDataset(
|
|
311
|
+
node_features=self.X,
|
|
312
|
+
edge_index=self.edge_index,
|
|
313
|
+
edge_weight=self.edge_weight,
|
|
314
|
+
node_labels=self.y, # 节点索引
|
|
315
|
+
device=self.device,
|
|
316
|
+
subgraph_size=subgraph_size
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
# 创建数据加载器
|
|
320
|
+
self.subgraph_loader = DataLoader(
|
|
321
|
+
self.subgraph_dataset,
|
|
322
|
+
batch_size=1, # 每次处理一个子图
|
|
323
|
+
shuffle=True,
|
|
324
|
+
num_workers=0
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
# 为了与原版本一致,我们也创建一个cdata-like的接口
|
|
328
|
+
self.cdata = self._create_cdata_interface()
|
|
329
|
+
|
|
330
|
+
def _create_cdata_interface(self):
|
|
331
|
+
"""
|
|
332
|
+
创建一个类似cdata的接口,用于与原始API保持一致
|
|
333
|
+
"""
|
|
334
|
+
class SubgraphIterator:
|
|
335
|
+
def __init__(self, subgraph_loader, num_subgraphs_per_epoch):
|
|
336
|
+
self.subgraph_loader = subgraph_loader
|
|
337
|
+
self.num_subgraphs_per_epoch = num_subgraphs_per_epoch
|
|
338
|
+
self._iterator = None
|
|
339
|
+
self._count = 0
|
|
340
|
+
|
|
341
|
+
def __iter__(self):
|
|
342
|
+
self._iterator = iter(self.subgraph_loader)
|
|
343
|
+
self._count = 0
|
|
344
|
+
return self
|
|
345
|
+
|
|
346
|
+
def __next__(self):
|
|
347
|
+
if self._count >= self.num_subgraphs_per_epoch:
|
|
348
|
+
raise StopIteration
|
|
349
|
+
|
|
350
|
+
try:
|
|
351
|
+
# DataLoader返回的是[batch],我们需要取第一个元素
|
|
352
|
+
batch = next(self._iterator)
|
|
353
|
+
if isinstance(batch, list) and len(batch) > 0:
|
|
354
|
+
data = batch[0] # 取第一个Data对象
|
|
355
|
+
else:
|
|
356
|
+
data = batch
|
|
357
|
+
|
|
358
|
+
self._count += 1
|
|
359
|
+
return data
|
|
360
|
+
except StopIteration:
|
|
361
|
+
raise StopIteration
|
|
362
|
+
|
|
363
|
+
return SubgraphIterator(self.subgraph_loader, self.num_subgraphs_per_epoch)
|
|
364
|
+
|
|
365
|
+
def step(self) -> None:
|
|
366
|
+
"""
|
|
367
|
+
执行一个训练步骤 - 与原版本API完全一致
|
|
368
|
+
"""
|
|
369
|
+
ls_l = []
|
|
370
|
+
original_indices = []
|
|
371
|
+
|
|
372
|
+
# 这里的逻辑与原版本完全一致:for cd in self.cdata
|
|
373
|
+
for cd in self.cdata:
|
|
374
|
+
# cd现在是一个torch_geometric.data.Data对象
|
|
375
|
+
self.update(cd) # 调用父类的update方法
|
|
376
|
+
latent = self.take_latent(cd) # 调用父类的take_latent方法
|
|
377
|
+
ls_l.append(latent)
|
|
378
|
+
|
|
379
|
+
# 保存原始节点索引用于重构
|
|
380
|
+
if hasattr(cd, 'original_node_idx'):
|
|
381
|
+
original_indices.append(cd.original_node_idx.cpu().numpy())
|
|
382
|
+
if original_indices:
|
|
383
|
+
self.idx = np.hstack(original_indices)
|
|
384
|
+
if ls_l:
|
|
385
|
+
# 重构完整的潜在表示
|
|
386
|
+
if original_indices:
|
|
387
|
+
full_latent = self._reconstruct_full_latent(ls_l, original_indices)
|
|
388
|
+
else:
|
|
389
|
+
# 如果没有索引信息,简单拼接(可能不完整)
|
|
390
|
+
full_latent = np.vstack(ls_l)
|
|
391
|
+
|
|
392
|
+
# 计算评分 - 与原版本一致
|
|
393
|
+
score = self._calc_score(full_latent)
|
|
394
|
+
self.score.append(score)
|
|
395
|
+
|
|
396
|
+
def step_ode(self) -> None:
|
|
397
|
+
"""
|
|
398
|
+
ODE版本的训练步骤 - 与原版本API完全一致
|
|
399
|
+
"""
|
|
400
|
+
ls_mix_latent1 = []
|
|
401
|
+
ls_mix_latent2 = []
|
|
402
|
+
original_indices = []
|
|
403
|
+
|
|
404
|
+
for cd in self.cdata:
|
|
405
|
+
self.update(cd) # 调用父类的update方法
|
|
406
|
+
latent1 = self.take_latent(cd) # 调用父类的take_latent方法
|
|
407
|
+
latent2 = self.take_odelatent(cd) # 调用父类的take_odelatent方法
|
|
408
|
+
|
|
409
|
+
ls_mix_latent1.append(latent1)
|
|
410
|
+
ls_mix_latent2.append(latent2)
|
|
411
|
+
|
|
412
|
+
if hasattr(cd, 'original_node_idx'):
|
|
413
|
+
original_indices.append(cd.original_node_idx.cpu().numpy())
|
|
414
|
+
if original_indices:
|
|
415
|
+
self.idx = np.hstack(original_indices)
|
|
416
|
+
if ls_mix_latent1 and ls_mix_latent2:
|
|
417
|
+
# 重构完整的潜在表示
|
|
418
|
+
if original_indices:
|
|
419
|
+
full_latent1 = self._reconstruct_full_latent(ls_mix_latent1, original_indices)
|
|
420
|
+
full_latent2 = self._reconstruct_full_latent(ls_mix_latent2, original_indices)
|
|
421
|
+
else:
|
|
422
|
+
full_latent1 = np.vstack(ls_mix_latent1)
|
|
423
|
+
full_latent2 = np.vstack(ls_mix_latent2)
|
|
424
|
+
|
|
425
|
+
# 混合潜在表示 - 与原版本一致
|
|
426
|
+
latent_mix = self.scale1 * full_latent1 + self.scale2 * full_latent2
|
|
427
|
+
|
|
428
|
+
score_mix = self._calc_score(latent_mix)
|
|
429
|
+
self.mix_score.append(score_mix)
|
|
430
|
+
|
|
431
|
+
def _reconstruct_full_latent(self, latent_list, indices_list):
|
|
432
|
+
"""
|
|
433
|
+
从子图的潜在表示重构完整的潜在表示
|
|
434
|
+
"""
|
|
435
|
+
if not latent_list:
|
|
436
|
+
return np.array([])
|
|
437
|
+
|
|
438
|
+
# 初始化完整的潜在表示矩阵
|
|
439
|
+
latent_dim = latent_list[0].shape[1]
|
|
440
|
+
full_latent = np.zeros((self.n_obs, latent_dim))
|
|
441
|
+
node_counts = np.zeros(self.n_obs)
|
|
442
|
+
|
|
443
|
+
# 累加所有子图的贡献
|
|
444
|
+
for latent, indices in zip(latent_list, indices_list):
|
|
445
|
+
for i, original_node_idx in enumerate(indices):
|
|
446
|
+
if 0 <= original_node_idx < self.n_obs:
|
|
447
|
+
full_latent[original_node_idx] += latent[i]
|
|
448
|
+
node_counts[original_node_idx] += 1
|
|
449
|
+
|
|
450
|
+
# 对被多次访问的节点取平均
|
|
451
|
+
for i in range(self.n_obs):
|
|
452
|
+
if node_counts[i] > 0:
|
|
453
|
+
full_latent[i] /= node_counts[i]
|
|
454
|
+
# 如果某个节点从未被访问,保持零向量(或者可以用其他策略)
|
|
455
|
+
|
|
456
|
+
return full_latent
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
class GNODEVAE_Env_r(GODEVAE_Trainer_r, envMixin, scMixin):
|
|
462
|
+
"""
|
|
463
|
+
Environment class for training and evaluating the GraphVAE model.
|
|
464
|
+
|
|
465
|
+
This class extends the `Trainer` and `envMixin` classes to provide functionality
|
|
466
|
+
for handling AnnData objects, clustering, and training the model in a batched manner
|
|
467
|
+
using `ClusterData`.
|
|
468
|
+
|
|
469
|
+
Parameters
|
|
470
|
+
----------
|
|
471
|
+
adata : AnnData
|
|
472
|
+
Annotated data matrix (e.g., single-cell data).
|
|
473
|
+
layer : str
|
|
474
|
+
Layer of the AnnData object to use for input features.
|
|
475
|
+
n_var : int
|
|
476
|
+
Number of highly variable genes to select.
|
|
477
|
+
tech : str
|
|
478
|
+
Decomposition method to use (PCA, NMF, FastICA, TruncatedSVD, FactorAnalysis, LatentDirichletAllocation).
|
|
479
|
+
n_neighbors : int
|
|
480
|
+
Number of neighbors for graph construction.
|
|
481
|
+
latent_dim : int
|
|
482
|
+
Latent space dimension for clustering.
|
|
483
|
+
batch_tech : Optional[str]
|
|
484
|
+
Method to correct batch effects ('harmony' or 'scvi').
|
|
485
|
+
all_feat : bool
|
|
486
|
+
Whether to use all features or only highly variable ones
|
|
487
|
+
hidden_dim : int
|
|
488
|
+
Hidden layer dimension for the encoder.
|
|
489
|
+
latent_dim : int
|
|
490
|
+
Latent space dimension.
|
|
491
|
+
encoder_type : str
|
|
492
|
+
Type of graph convolutional layer ('GCN', 'Cheb', 'SAGE', 'Graph', 'TAG', 'ARMA', 'GAT', 'Transformer', 'SG', 'SSG').
|
|
493
|
+
encoder_hidden_layers : int
|
|
494
|
+
Number of hidden layers in the graph encoder.
|
|
495
|
+
decoder_type : str
|
|
496
|
+
Type of graph decoder ('Bilinear', 'InnerProduct', 'MLP').
|
|
497
|
+
decoder_hidden_dim : int
|
|
498
|
+
Hidden dimension for the MLPDecoder (if used).
|
|
499
|
+
feature_decoder_hidden_layers : int
|
|
500
|
+
Number of hidden layers in the feature decoder.
|
|
501
|
+
dropout : float
|
|
502
|
+
Dropout rate.
|
|
503
|
+
use_residual : bool, optional
|
|
504
|
+
Whether to use residual connections, by default True.
|
|
505
|
+
Cheb_k : int, optional
|
|
506
|
+
The order of Chebyshev polynomials for ChebConv, by default None.
|
|
507
|
+
alpha : float, optional
|
|
508
|
+
Teleport probability, by default 0.5.
|
|
509
|
+
lr : float
|
|
510
|
+
Learning rate for the optimizer.
|
|
511
|
+
beta : float
|
|
512
|
+
Weight for the KL divergence term in the loss function.
|
|
513
|
+
graph : float
|
|
514
|
+
Weight for the graph reconstruction loss.
|
|
515
|
+
device : torch.device
|
|
516
|
+
Device to run the model on (e.g., 'cpu' or 'cuda').
|
|
517
|
+
num_parts : int
|
|
518
|
+
Number of partitions for clustering the graph data.
|
|
519
|
+
*args : tuple
|
|
520
|
+
Additional positional arguments.
|
|
521
|
+
**kwargs : dict
|
|
522
|
+
Additional keyword arguments.
|
|
523
|
+
|
|
524
|
+
Attributes
|
|
525
|
+
----------
|
|
526
|
+
X : np.ndarray
|
|
527
|
+
Input feature matrix (log-transformed).
|
|
528
|
+
n_obs : int
|
|
529
|
+
Number of observations (nodes).
|
|
530
|
+
n_var : int
|
|
531
|
+
Number of variables (features).
|
|
532
|
+
labels : np.ndarray
|
|
533
|
+
Cluster labels for the input data.
|
|
534
|
+
edge_index : np.ndarray
|
|
535
|
+
Edge indices of the graph.
|
|
536
|
+
edge_weight : np.ndarray
|
|
537
|
+
Edge weights of the graph.
|
|
538
|
+
y : np.ndarray
|
|
539
|
+
Node indices.
|
|
540
|
+
cdata : ClusterData
|
|
541
|
+
Clustered graph data for batched training.
|
|
542
|
+
idx : np.ndarray
|
|
543
|
+
Node indices for all clusters.
|
|
544
|
+
score : List[Tuple[float, float, float, float, float, float]]
|
|
545
|
+
List of clustering and correlation scores for each training step.
|
|
546
|
+
"""
|
|
547
|
+
|
|
548
|
+
def __init__(
|
|
549
|
+
self,
|
|
550
|
+
adata: AnnData,
|
|
551
|
+
layer: str,
|
|
552
|
+
n_var: int,
|
|
553
|
+
tech: str,
|
|
554
|
+
n_neighbors: int,
|
|
555
|
+
batch_tech: Optional[str],
|
|
556
|
+
all_feat: bool,
|
|
557
|
+
hidden_dim: int,
|
|
558
|
+
latent_dim: int,
|
|
559
|
+
ode_hidden_dim: int,
|
|
560
|
+
encoder_type: str,
|
|
561
|
+
graph_type: str,
|
|
562
|
+
structure_decoder_type: str,
|
|
563
|
+
feature_decoder_type: str,
|
|
564
|
+
hidden_layers: int,
|
|
565
|
+
decoder_hidden_dim: int,
|
|
566
|
+
dropout: float,
|
|
567
|
+
use_residual: bool,
|
|
568
|
+
Cheb_k: int,
|
|
569
|
+
alpha: float,
|
|
570
|
+
threshold: float,
|
|
571
|
+
sparse_threshold: Optional[int],
|
|
572
|
+
lr: float,
|
|
573
|
+
beta: float,
|
|
574
|
+
graph: float,
|
|
575
|
+
w_recon: float,
|
|
576
|
+
w_kl: float ,
|
|
577
|
+
w_adj: float,
|
|
578
|
+
w_recon_ode: float,
|
|
579
|
+
w_z_div: float,
|
|
580
|
+
device: torch.device,
|
|
581
|
+
num_parts: int,
|
|
582
|
+
latent_type: str,
|
|
583
|
+
*args,
|
|
584
|
+
**kwargs,
|
|
585
|
+
):
|
|
586
|
+
self._register_adata(adata, layer, n_var, tech, n_neighbors, latent_dim, batch_tech, all_feat)
|
|
587
|
+
super().__init__(
|
|
588
|
+
self.n_var,
|
|
589
|
+
hidden_dim,
|
|
590
|
+
latent_dim,
|
|
591
|
+
ode_hidden_dim,
|
|
592
|
+
encoder_type,
|
|
593
|
+
graph_type,
|
|
594
|
+
structure_decoder_type,
|
|
595
|
+
feature_decoder_type,
|
|
596
|
+
hidden_layers,
|
|
597
|
+
decoder_hidden_dim,
|
|
598
|
+
dropout,
|
|
599
|
+
use_residual,
|
|
600
|
+
Cheb_k,
|
|
601
|
+
alpha,
|
|
602
|
+
threshold,
|
|
603
|
+
sparse_threshold,
|
|
604
|
+
lr,
|
|
605
|
+
beta,
|
|
606
|
+
graph,
|
|
607
|
+
w_recon,
|
|
608
|
+
w_kl,
|
|
609
|
+
w_adj,
|
|
610
|
+
w_recon_ode,
|
|
611
|
+
w_z_div,
|
|
612
|
+
device,
|
|
613
|
+
latent_type,
|
|
614
|
+
)
|
|
615
|
+
self._register_cdata(num_parts)
|
|
616
|
+
self.score: List[Tuple[float, float, float, float, float, float]] = []
|
|
617
|
+
self.mix_score: List[Tuple[float, float, float, float, float, float]] = []
|
|
618
|
+
|
|
619
|
+
def step(self) -> None:
|
|
620
|
+
"""
|
|
621
|
+
Perform a single training step.
|
|
622
|
+
|
|
623
|
+
This method iterates over the clustered data, updates the model parameters,
|
|
624
|
+
computes the latent representation, and calculates clustering and correlation scores.
|
|
625
|
+
|
|
626
|
+
Returns
|
|
627
|
+
-------
|
|
628
|
+
None
|
|
629
|
+
"""
|
|
630
|
+
ls_l = []
|
|
631
|
+
for cd in self.cdata:
|
|
632
|
+
self.update(cd)
|
|
633
|
+
latent = self.take_latent(cd)
|
|
634
|
+
ls_l.append(latent)
|
|
635
|
+
latent = np.vstack(ls_l)
|
|
636
|
+
score = self._calc_score(latent)
|
|
637
|
+
self.score.append(score)
|
|
638
|
+
|
|
639
|
+
def step_ode(self) -> None:
|
|
640
|
+
ls_mix = []
|
|
641
|
+
for cd in self.cdata:
|
|
642
|
+
self.update(cd)
|
|
643
|
+
latent1 = self.take_latent(cd)
|
|
644
|
+
latent2 = self.take_odelatent(cd)
|
|
645
|
+
latent = self.scale1 * latent1 + self.scale2 * latent2
|
|
646
|
+
ls_mix.append(latent)
|
|
647
|
+
|
|
648
|
+
latent_mix = np.vstack(ls_mix)
|
|
649
|
+
|
|
650
|
+
score_mix = self._calc_score(latent_mix)
|
|
651
|
+
|
|
652
|
+
self.mix_score.append(score_mix)
|
|
653
|
+
|
|
654
|
+
def _register_adata(
|
|
655
|
+
self,
|
|
656
|
+
adata: AnnData,
|
|
657
|
+
layer: str,
|
|
658
|
+
n_var: int,
|
|
659
|
+
tech: str,
|
|
660
|
+
n_neighbors: int,
|
|
661
|
+
latent_dim: int,
|
|
662
|
+
batch_tech: Optional[str],
|
|
663
|
+
all_feat: bool
|
|
664
|
+
) -> None:
|
|
665
|
+
"""
|
|
666
|
+
Register AnnData object and preprocess the data.
|
|
667
|
+
|
|
668
|
+
This method extracts the input features, graph structure, and cluster labels
|
|
669
|
+
from the AnnData object.
|
|
670
|
+
|
|
671
|
+
Parameters
|
|
672
|
+
----------
|
|
673
|
+
adata : AnnData
|
|
674
|
+
Annotated data matrix (e.g., single-cell data).
|
|
675
|
+
layer : str
|
|
676
|
+
Layer of the AnnData object to store original features.
|
|
677
|
+
n_var : int
|
|
678
|
+
Number of highly variable genes to select.
|
|
679
|
+
tech : str
|
|
680
|
+
Decomposition method to use (PCA, NMF, FastICA, TruncatedSVD, FactorAnalysis, LatentDirichletAllocation).
|
|
681
|
+
n_neighbors : int
|
|
682
|
+
Number of neighbors for graph construction.
|
|
683
|
+
latent_dim : int
|
|
684
|
+
Latent space dimension for clustering.
|
|
685
|
+
batch_tech : Optional[str]
|
|
686
|
+
Method to correct batch effects ('harmony' or 'scvi').
|
|
687
|
+
all_feat : bool
|
|
688
|
+
Whether to use all features or only highly variable ones.
|
|
689
|
+
|
|
690
|
+
Returns
|
|
691
|
+
-------
|
|
692
|
+
None
|
|
693
|
+
"""
|
|
694
|
+
self._preprocess(adata, layer, n_var)
|
|
695
|
+
self._decomposition(adata, tech, latent_dim)
|
|
696
|
+
|
|
697
|
+
if batch_tech:
|
|
698
|
+
self._batchcorrect(adata, batch_tech, tech, layer)
|
|
699
|
+
|
|
700
|
+
# Determine the representation to use for neighborhood graph
|
|
701
|
+
if batch_tech == 'harmony':
|
|
702
|
+
use_rep = f'X_harmony_{tech}'
|
|
703
|
+
elif batch_tech == 'scvi':
|
|
704
|
+
use_rep = 'X_scvi'
|
|
705
|
+
else:
|
|
706
|
+
use_rep = f'X_{tech}'
|
|
707
|
+
|
|
708
|
+
# Construct the neighborhood graph
|
|
709
|
+
sc.pp.neighbors(adata, n_neighbors=n_neighbors, use_rep=use_rep)
|
|
710
|
+
|
|
711
|
+
# Select features
|
|
712
|
+
if all_feat:
|
|
713
|
+
self.X = np.log1p(adata.layers[layer].toarray())
|
|
714
|
+
else:
|
|
715
|
+
self.X = adata[:, adata.var['highly_variable']].X.toarray()
|
|
716
|
+
|
|
717
|
+
self.n_obs, self.n_var = self.X.shape
|
|
718
|
+
|
|
719
|
+
# Perform clustering
|
|
720
|
+
self.labels = KMeans(n_clusters=latent_dim).fit_predict(self.X)
|
|
721
|
+
|
|
722
|
+
# Extract graph information
|
|
723
|
+
coo = adata.obsp['connectivities'].tocoo()
|
|
724
|
+
self.edge_index = np.array([coo.row, coo.col])
|
|
725
|
+
self.edge_weight = coo.data
|
|
726
|
+
|
|
727
|
+
# Assign indices
|
|
728
|
+
self.y = np.arange(adata.shape[0])
|
|
729
|
+
|
|
730
|
+
def _register_cdata(
|
|
731
|
+
self,
|
|
732
|
+
num_parts: int,
|
|
733
|
+
) -> None:
|
|
734
|
+
"""
|
|
735
|
+
Register clustered graph data for batched training.
|
|
736
|
+
|
|
737
|
+
This method partitions the graph data into clusters for efficient training.
|
|
738
|
+
|
|
739
|
+
Parameters
|
|
740
|
+
----------
|
|
741
|
+
num_parts : int
|
|
742
|
+
Number of partitions for clustering the graph data.
|
|
743
|
+
|
|
744
|
+
Returns
|
|
745
|
+
-------
|
|
746
|
+
None
|
|
747
|
+
"""
|
|
748
|
+
data = Data(
|
|
749
|
+
x=torch.tensor(self.X, dtype=torch.float, device=self.device),
|
|
750
|
+
edge_index=torch.tensor(self.edge_index, dtype=torch.long, device=self.device),
|
|
751
|
+
edge_attr=torch.tensor(self.edge_weight, dtype=torch.float, device=self.device),
|
|
752
|
+
y=torch.tensor(self.y, dtype=torch.long, device=self.device),
|
|
753
|
+
)
|
|
754
|
+
self.cdata = ClusterData(data, num_parts=num_parts)
|
|
755
|
+
ls_y = []
|
|
756
|
+
for cd in self.cdata:
|
|
757
|
+
ls_y.append(cd.y.cpu().numpy())
|
|
758
|
+
self.idx = np.hstack(ls_y)
|