iaode 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- iaode/BEN.py +435 -0
- iaode/DRE.py +427 -0
- iaode/LSE.py +635 -0
- iaode/__init__.py +97 -0
- iaode/agent.py +190 -0
- iaode/annotation.py +588 -0
- iaode/environment.py +268 -0
- iaode/mixin.py +353 -0
- iaode/model.py +634 -0
- iaode/module.py +473 -0
- iaode/utils.py +326 -0
- iaode-0.1.0.dist-info/METADATA +373 -0
- iaode-0.1.0.dist-info/RECORD +16 -0
- iaode-0.1.0.dist-info/WHEEL +5 -0
- iaode-0.1.0.dist-info/licenses/LICENSE +21 -0
- iaode-0.1.0.dist-info/top_level.txt +1 -0
iaode/BEN.py
ADDED
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
import gc
|
|
2
|
+
import os
|
|
3
|
+
import time
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import scanpy as sc
|
|
7
|
+
import scipy.sparse as sp
|
|
8
|
+
import torch
|
|
9
|
+
from sklearn.cluster import KMeans
|
|
10
|
+
from sklearn.metrics import (
|
|
11
|
+
normalized_mutual_info_score,
|
|
12
|
+
adjusted_mutual_info_score,
|
|
13
|
+
silhouette_score,
|
|
14
|
+
davies_bouldin_score,
|
|
15
|
+
calinski_harabasz_score
|
|
16
|
+
)
|
|
17
|
+
from sklearn.model_selection import train_test_split
|
|
18
|
+
|
|
19
|
+
# scVI imports
|
|
20
|
+
from scvi.model import SCVI, PEAKVI
|
|
21
|
+
from scvi.external import POISSONVI
|
|
22
|
+
from .DRE import evaluate_dimensionality_reduction
|
|
23
|
+
from .LSE import evaluate_single_cell_latent_space
|
|
24
|
+
|
|
25
|
+
class DataSplitter:
|
|
26
|
+
"""
|
|
27
|
+
Simplified data splitter for consistent train/val/test splits across all models.
|
|
28
|
+
|
|
29
|
+
Strategy: 70% train, 15% val, 15% test
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(self, n_samples, test_size=0.15, val_size=0.15, random_state=42):
|
|
33
|
+
"""
|
|
34
|
+
Parameters
|
|
35
|
+
----------
|
|
36
|
+
n_samples : int
|
|
37
|
+
Total number of samples
|
|
38
|
+
test_size : float
|
|
39
|
+
Proportion for test set (default 0.15)
|
|
40
|
+
val_size : float
|
|
41
|
+
Proportion for validation set (default 0.15)
|
|
42
|
+
random_state : int
|
|
43
|
+
Random seed for reproducibility
|
|
44
|
+
"""
|
|
45
|
+
self.n_samples = n_samples
|
|
46
|
+
self.test_size = test_size
|
|
47
|
+
self.val_size = val_size
|
|
48
|
+
self.random_state = random_state
|
|
49
|
+
|
|
50
|
+
self.train_val_size = 1 - test_size
|
|
51
|
+
self.val_size_relative = val_size / self.train_val_size
|
|
52
|
+
|
|
53
|
+
self._create_splits()
|
|
54
|
+
|
|
55
|
+
def _create_splits(self):
|
|
56
|
+
"""Create train/val/test indices"""
|
|
57
|
+
indices = np.arange(self.n_samples)
|
|
58
|
+
|
|
59
|
+
# First split: separate test set (15%)
|
|
60
|
+
train_val_idx, test_idx = train_test_split(
|
|
61
|
+
indices,
|
|
62
|
+
test_size=self.test_size,
|
|
63
|
+
random_state=self.random_state,
|
|
64
|
+
shuffle=True
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
# Second split: separate train and val from remaining 85%
|
|
68
|
+
train_idx, val_idx = train_test_split(
|
|
69
|
+
train_val_idx,
|
|
70
|
+
test_size=self.val_size_relative,
|
|
71
|
+
random_state=self.random_state,
|
|
72
|
+
shuffle=True
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
self.train_idx = train_idx
|
|
76
|
+
self.val_idx = val_idx
|
|
77
|
+
self.test_idx = test_idx
|
|
78
|
+
self.train_val_idx = train_val_idx
|
|
79
|
+
|
|
80
|
+
print(f"\nData split sizes:")
|
|
81
|
+
print(f" Total: {self.n_samples}")
|
|
82
|
+
print(f" Train: {len(train_idx)} ({len(train_idx)/self.n_samples*100:.1f}%)")
|
|
83
|
+
print(f" Val: {len(val_idx)} ({len(val_idx)/self.n_samples*100:.1f}%)")
|
|
84
|
+
print(f" Test: {len(test_idx)} ({len(test_idx)/self.n_samples*100:.1f}%)")
|
|
85
|
+
|
|
86
|
+
def get_scvi_validation_size(self):
|
|
87
|
+
"""Get validation size for scVI's internal split"""
|
|
88
|
+
return self.val_size_relative
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def evaluate_model(latent, labels, adata_subset):
|
|
92
|
+
"""
|
|
93
|
+
Comprehensive evaluation of a single model's latent representation.
|
|
94
|
+
|
|
95
|
+
Parameters
|
|
96
|
+
----------
|
|
97
|
+
latent : np.ndarray
|
|
98
|
+
Latent representation (n_cells, n_latent)
|
|
99
|
+
adata_subset : AnnData
|
|
100
|
+
AnnData object for the test set
|
|
101
|
+
|
|
102
|
+
Returns
|
|
103
|
+
-------
|
|
104
|
+
dict
|
|
105
|
+
Dictionary of evaluation metrics
|
|
106
|
+
"""
|
|
107
|
+
# Clustering evaluation
|
|
108
|
+
n_clusters = len(np.unique(labels))
|
|
109
|
+
pred_label = KMeans(n_clusters, random_state=42).fit_predict(latent)
|
|
110
|
+
|
|
111
|
+
nmi = normalized_mutual_info_score(labels, pred_label)
|
|
112
|
+
ari = adjusted_mutual_info_score(labels, pred_label)
|
|
113
|
+
asw = silhouette_score(latent, pred_label)
|
|
114
|
+
dav = davies_bouldin_score(latent, pred_label)
|
|
115
|
+
cal = calinski_harabasz_score(latent, pred_label)
|
|
116
|
+
|
|
117
|
+
# Correlation in latent space
|
|
118
|
+
acorr = abs(np.corrcoef(latent.T))
|
|
119
|
+
cor = acorr.sum(axis=1).mean().item() - 1
|
|
120
|
+
|
|
121
|
+
# Dimensionality reduction evaluation
|
|
122
|
+
adata_subset.obsm['X_latent'] = latent
|
|
123
|
+
|
|
124
|
+
# UMAP
|
|
125
|
+
sc.pp.neighbors(adata_subset, use_rep='X_latent')
|
|
126
|
+
sc.tl.umap(adata_subset)
|
|
127
|
+
X_umap = adata_subset.obsm['X_umap']
|
|
128
|
+
res_umap = evaluate_dimensionality_reduction(latent, X_umap)
|
|
129
|
+
|
|
130
|
+
# t-SNE
|
|
131
|
+
sc.tl.tsne(adata_subset, use_rep='X_latent')
|
|
132
|
+
X_tsne = adata_subset.obsm['X_tsne']
|
|
133
|
+
res_tsne = evaluate_dimensionality_reduction(latent, X_tsne)
|
|
134
|
+
|
|
135
|
+
# Intrinsic evaluation
|
|
136
|
+
res_intrin = evaluate_single_cell_latent_space(latent)
|
|
137
|
+
|
|
138
|
+
# Combine all metrics
|
|
139
|
+
metrics = {
|
|
140
|
+
'NMI': nmi,
|
|
141
|
+
'ARI': ari,
|
|
142
|
+
'ASW': asw,
|
|
143
|
+
'DAV': dav,
|
|
144
|
+
'CAL': cal,
|
|
145
|
+
'COR': cor,
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
# Add UMAP metrics
|
|
149
|
+
for key, val in res_umap.items():
|
|
150
|
+
metrics[f"{key}_umap"] = val
|
|
151
|
+
|
|
152
|
+
# Add t-SNE metrics
|
|
153
|
+
for key, val in res_tsne.items():
|
|
154
|
+
metrics[f"{key}_tsne"] = val
|
|
155
|
+
|
|
156
|
+
# Add intrinsic metrics
|
|
157
|
+
for key, val in res_intrin.items():
|
|
158
|
+
metrics[f"{key}_intrin"] = val
|
|
159
|
+
|
|
160
|
+
return metrics
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def train_scvi_models(adata, splitter, n_latent=10, n_epochs=400, batch_size=128):
|
|
164
|
+
"""
|
|
165
|
+
Train scVI-architecture models (SCVI, PEAKVI, POISSONVI) with consistent splits.
|
|
166
|
+
|
|
167
|
+
Parameters
|
|
168
|
+
----------
|
|
169
|
+
adata : AnnData
|
|
170
|
+
Full dataset
|
|
171
|
+
splitter : DataSplitter
|
|
172
|
+
Data splitter instance
|
|
173
|
+
n_latent : int
|
|
174
|
+
Latent dimension
|
|
175
|
+
n_epochs : int
|
|
176
|
+
Number of training epochs
|
|
177
|
+
|
|
178
|
+
Returns
|
|
179
|
+
-------
|
|
180
|
+
dict
|
|
181
|
+
Dictionary containing trained models and their test data
|
|
182
|
+
"""
|
|
183
|
+
results = {}
|
|
184
|
+
use_cuda = torch.cuda.is_available()
|
|
185
|
+
|
|
186
|
+
# Prepare data: scVI sees only train+val (85%)
|
|
187
|
+
adata_trainval = adata[splitter.train_val_idx].copy()
|
|
188
|
+
adata_test = adata[splitter.test_idx].copy()
|
|
189
|
+
|
|
190
|
+
validation_size = splitter.get_scvi_validation_size()
|
|
191
|
+
|
|
192
|
+
# ==================== SCVI ====================
|
|
193
|
+
print(f"\n{'='*70}")
|
|
194
|
+
print("Training SCVI Model")
|
|
195
|
+
print(f"{'='*70}")
|
|
196
|
+
|
|
197
|
+
try:
|
|
198
|
+
# Reset GPU memory stats and start timer
|
|
199
|
+
if use_cuda:
|
|
200
|
+
torch.cuda.reset_peak_memory_stats()
|
|
201
|
+
start_time = time.time()
|
|
202
|
+
|
|
203
|
+
# Setup for SCVI (uses raw counts from layers)
|
|
204
|
+
SCVI.setup_anndata(
|
|
205
|
+
adata_trainval,
|
|
206
|
+
layer="counts",
|
|
207
|
+
batch_key=None
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
scvi_model = SCVI(
|
|
211
|
+
adata_trainval,
|
|
212
|
+
n_latent=n_latent,
|
|
213
|
+
n_hidden=128,
|
|
214
|
+
n_layers=2,
|
|
215
|
+
dropout_rate=0.1,
|
|
216
|
+
gene_likelihood="nb"
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
scvi_model.train(
|
|
220
|
+
max_epochs=n_epochs,
|
|
221
|
+
train_size=1 - validation_size,
|
|
222
|
+
validation_size=validation_size,
|
|
223
|
+
early_stopping=True,
|
|
224
|
+
early_stopping_patience=20,
|
|
225
|
+
check_val_every_n_epoch=5,
|
|
226
|
+
batch_size=batch_size,
|
|
227
|
+
plan_kwargs={'lr': 1e-4}
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
# Record time and GPU memory
|
|
231
|
+
train_time = time.time() - start_time
|
|
232
|
+
peak_memory = torch.cuda.max_memory_allocated() / 1e9 if use_cuda else 0
|
|
233
|
+
actual_epochs = len(scvi_model.history['elbo_train']) # Actual epochs trained
|
|
234
|
+
|
|
235
|
+
# Setup test data
|
|
236
|
+
SCVI.setup_anndata(adata_test, layer="counts", batch_key=None)
|
|
237
|
+
|
|
238
|
+
results['scvi'] = {
|
|
239
|
+
'model': scvi_model,
|
|
240
|
+
'adata_test': adata_test.copy(),
|
|
241
|
+
'history': scvi_model.history,
|
|
242
|
+
'train_time': train_time,
|
|
243
|
+
'peak_memory_gb': peak_memory,
|
|
244
|
+
'actual_epochs': actual_epochs
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
print(f"✓ SCVI training completed")
|
|
248
|
+
print(f" Epochs: {actual_epochs}/{n_epochs}, Time: {train_time:.2f}s, Peak GPU Memory: {peak_memory:.3f} GB")
|
|
249
|
+
|
|
250
|
+
except Exception as e:
|
|
251
|
+
print(f"✗ SCVI training failed: {str(e)}")
|
|
252
|
+
results['scvi'] = None
|
|
253
|
+
|
|
254
|
+
# ==================== PEAKVI ====================
|
|
255
|
+
print(f"\n{'='*70}")
|
|
256
|
+
print("Training PEAKVI Model")
|
|
257
|
+
print(f"{'='*70}")
|
|
258
|
+
|
|
259
|
+
try:
|
|
260
|
+
# Reset GPU memory stats and start timer
|
|
261
|
+
if use_cuda:
|
|
262
|
+
torch.cuda.reset_peak_memory_stats()
|
|
263
|
+
start_time = time.time()
|
|
264
|
+
|
|
265
|
+
# Reset adata for PEAKVI
|
|
266
|
+
adata_trainval = adata[splitter.train_val_idx].copy()
|
|
267
|
+
|
|
268
|
+
PEAKVI.setup_anndata(
|
|
269
|
+
adata_trainval,
|
|
270
|
+
layer="counts",
|
|
271
|
+
batch_key=None
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
peakvi_model = PEAKVI(
|
|
275
|
+
adata_trainval,
|
|
276
|
+
n_latent=n_latent,
|
|
277
|
+
n_hidden=128,
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
peakvi_model.train(
|
|
281
|
+
max_epochs=n_epochs,
|
|
282
|
+
train_size=1 - validation_size,
|
|
283
|
+
validation_size=validation_size,
|
|
284
|
+
early_stopping=True,
|
|
285
|
+
early_stopping_patience=20,
|
|
286
|
+
check_val_every_n_epoch=5,
|
|
287
|
+
batch_size=batch_size,
|
|
288
|
+
plan_kwargs={'lr': 1e-4}
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
# Record time and GPU memory
|
|
292
|
+
train_time = time.time() - start_time
|
|
293
|
+
peak_memory = torch.cuda.max_memory_allocated() / 1e9 if use_cuda else 0
|
|
294
|
+
actual_epochs = len(peakvi_model.history['elbo_train'])
|
|
295
|
+
|
|
296
|
+
# Setup test data
|
|
297
|
+
adata_test_peakvi = adata[splitter.test_idx].copy()
|
|
298
|
+
PEAKVI.setup_anndata(adata_test_peakvi, layer="counts", batch_key=None)
|
|
299
|
+
|
|
300
|
+
results['peakvi'] = {
|
|
301
|
+
'model': peakvi_model,
|
|
302
|
+
'adata_test': adata_test_peakvi,
|
|
303
|
+
'history': peakvi_model.history,
|
|
304
|
+
'train_time': train_time,
|
|
305
|
+
'peak_memory_gb': peak_memory,
|
|
306
|
+
'actual_epochs': actual_epochs
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
print(f"✓ PEAKVI training completed")
|
|
310
|
+
print(f" Epochs: {actual_epochs}/{n_epochs}, Time: {train_time:.2f}s, Peak GPU Memory: {peak_memory:.3f} GB")
|
|
311
|
+
|
|
312
|
+
except Exception as e:
|
|
313
|
+
print(f"✗ PEAKVI training failed: {str(e)}")
|
|
314
|
+
results['peakvi'] = None
|
|
315
|
+
|
|
316
|
+
# ==================== POISSONVI ====================
|
|
317
|
+
print(f"\n{'='*70}")
|
|
318
|
+
print("Training POISSONVI Model")
|
|
319
|
+
print(f"{'='*70}")
|
|
320
|
+
|
|
321
|
+
try:
|
|
322
|
+
# Reset GPU memory stats and start timer
|
|
323
|
+
if use_cuda:
|
|
324
|
+
torch.cuda.reset_peak_memory_stats()
|
|
325
|
+
start_time = time.time()
|
|
326
|
+
|
|
327
|
+
# Reset adata for POISSONVI
|
|
328
|
+
adata_trainval = adata[splitter.train_val_idx].copy()
|
|
329
|
+
|
|
330
|
+
POISSONVI.setup_anndata(
|
|
331
|
+
adata_trainval,
|
|
332
|
+
layer="counts",
|
|
333
|
+
batch_key=None
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
poissonvi_model = POISSONVI(
|
|
337
|
+
adata_trainval,
|
|
338
|
+
n_latent=n_latent,
|
|
339
|
+
n_hidden=128,
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
poissonvi_model.train(
|
|
343
|
+
max_epochs=n_epochs,
|
|
344
|
+
train_size=1 - validation_size,
|
|
345
|
+
validation_size=validation_size,
|
|
346
|
+
early_stopping=True,
|
|
347
|
+
early_stopping_patience=20,
|
|
348
|
+
check_val_every_n_epoch=5,
|
|
349
|
+
batch_size=batch_size,
|
|
350
|
+
plan_kwargs={'lr': 1e-4}
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
# Record time and GPU memory
|
|
354
|
+
train_time = time.time() - start_time
|
|
355
|
+
peak_memory = torch.cuda.max_memory_allocated() / 1e9 if use_cuda else 0
|
|
356
|
+
actual_epochs = len(poissonvi_model.history['elbo_train'])
|
|
357
|
+
|
|
358
|
+
# Setup test data
|
|
359
|
+
adata_test_poissonvi = adata[splitter.test_idx].copy()
|
|
360
|
+
POISSONVI.setup_anndata(adata_test_poissonvi, layer="counts", batch_key=None)
|
|
361
|
+
|
|
362
|
+
results['poissonvi'] = {
|
|
363
|
+
'model': poissonvi_model,
|
|
364
|
+
'adata_test': adata_test_poissonvi,
|
|
365
|
+
'history': poissonvi_model.history,
|
|
366
|
+
'train_time': train_time,
|
|
367
|
+
'peak_memory_gb': peak_memory,
|
|
368
|
+
'actual_epochs': actual_epochs
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
print(f"✓ POISSONVI training completed")
|
|
372
|
+
print(f" Epochs: {actual_epochs}/{n_epochs}, Time: {train_time:.2f}s, Peak GPU Memory: {peak_memory:.3f} GB")
|
|
373
|
+
|
|
374
|
+
except Exception as e:
|
|
375
|
+
print(f"✗ POISSONVI training failed: {str(e)}")
|
|
376
|
+
results['poissonvi'] = None
|
|
377
|
+
|
|
378
|
+
return results
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def evaluate_scvi_models(scvi_results, adata_full, test_idx):
|
|
382
|
+
"""
|
|
383
|
+
Evaluate scVI-architecture models on test set.
|
|
384
|
+
|
|
385
|
+
Parameters
|
|
386
|
+
----------
|
|
387
|
+
scvi_results : dict
|
|
388
|
+
Results from train_scvi_models
|
|
389
|
+
adata_full : AnnData
|
|
390
|
+
Full dataset (for creating test subset)
|
|
391
|
+
test_idx : np.ndarray
|
|
392
|
+
Test set indices
|
|
393
|
+
|
|
394
|
+
Returns
|
|
395
|
+
-------
|
|
396
|
+
dict
|
|
397
|
+
Evaluation metrics for each model
|
|
398
|
+
"""
|
|
399
|
+
results = {}
|
|
400
|
+
|
|
401
|
+
for model_name, result in scvi_results.items():
|
|
402
|
+
if result is None:
|
|
403
|
+
print(f"\n⊗ Skipping {model_name.upper()} (training failed)")
|
|
404
|
+
results[model_name] = None
|
|
405
|
+
continue
|
|
406
|
+
|
|
407
|
+
print(f"\n{'='*70}")
|
|
408
|
+
print(f"Evaluating {model_name.upper()}")
|
|
409
|
+
print(f"{'='*70}")
|
|
410
|
+
|
|
411
|
+
try:
|
|
412
|
+
model = result['model']
|
|
413
|
+
adata_test = result['adata_test']
|
|
414
|
+
|
|
415
|
+
# Get latent representation
|
|
416
|
+
latent = model.get_latent_representation(adata_test)
|
|
417
|
+
|
|
418
|
+
# Create test subset for visualization
|
|
419
|
+
adata_test_subset = adata_full[test_idx].copy()
|
|
420
|
+
|
|
421
|
+
# KMeans labels
|
|
422
|
+
labels = KMeans(latent.shape[1]).fit_predict(adata_test_subset.layers['counts'])
|
|
423
|
+
|
|
424
|
+
# Evaluate
|
|
425
|
+
metrics = evaluate_model(latent, labels, adata_test_subset)
|
|
426
|
+
results[model_name] = metrics
|
|
427
|
+
|
|
428
|
+
print(f"✓ {model_name.upper()} evaluation completed")
|
|
429
|
+
print(f" NMI: {metrics['NMI']:.4f}, ARI: {metrics['ARI']:.4f}, ASW: {metrics['ASW']:.4f}")
|
|
430
|
+
|
|
431
|
+
except Exception as e:
|
|
432
|
+
print(f"✗ {model_name.upper()} evaluation failed: {str(e)}")
|
|
433
|
+
results[model_name] = None
|
|
434
|
+
|
|
435
|
+
return results
|