iaode 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- iaode/BEN.py +438 -0
- iaode/DRE.py +430 -0
- iaode/LSE.py +635 -0
- iaode/__init__.py +103 -0
- iaode/agent.py +850 -0
- iaode/annotation.py +592 -0
- iaode/datasets.py +330 -0
- iaode/environment.py +298 -0
- iaode/mixin.py +359 -0
- iaode/model.py +486 -0
- iaode/module.py +576 -0
- iaode/utils.py +327 -0
- iaode-0.2.1.dist-info/METADATA +697 -0
- iaode-0.2.1.dist-info/RECORD +17 -0
- iaode-0.2.1.dist-info/WHEEL +5 -0
- iaode-0.2.1.dist-info/licenses/LICENSE +21 -0
- iaode-0.2.1.dist-info/top_level.txt +1 -0
iaode/datasets.py
ADDED
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Dataset utilities for iAODE examples.
|
|
3
|
+
|
|
4
|
+
Provides automatic download and caching of example datasets,
|
|
5
|
+
including scATAC-seq data and reference annotations.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import urllib.request
|
|
10
|
+
import gzip
|
|
11
|
+
import shutil
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Optional, Tuple
|
|
14
|
+
import anndata as ad
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def get_data_dir() -> Path:
|
|
18
|
+
"""
|
|
19
|
+
Get the iAODE data cache directory.
|
|
20
|
+
|
|
21
|
+
Creates directory structure:
|
|
22
|
+
~/.iaode/data/
|
|
23
|
+
├── atacseq/
|
|
24
|
+
└── annotations/
|
|
25
|
+
|
|
26
|
+
Returns
|
|
27
|
+
-------
|
|
28
|
+
Path
|
|
29
|
+
Path to data cache directory
|
|
30
|
+
"""
|
|
31
|
+
data_dir = Path.home() / ".iaode" / "data"
|
|
32
|
+
data_dir.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
(data_dir / "atacseq").mkdir(exist_ok=True)
|
|
34
|
+
(data_dir / "annotations").mkdir(exist_ok=True)
|
|
35
|
+
return data_dir
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _download_file(url: str, output_path: Path, desc: str = "file") -> None:
|
|
39
|
+
"""
|
|
40
|
+
Download a file with progress indication.
|
|
41
|
+
|
|
42
|
+
Parameters
|
|
43
|
+
----------
|
|
44
|
+
url : str
|
|
45
|
+
URL to download from
|
|
46
|
+
output_path : Path
|
|
47
|
+
Local path to save file
|
|
48
|
+
desc : str
|
|
49
|
+
Description for progress messages
|
|
50
|
+
"""
|
|
51
|
+
print(f"📥 Downloading {desc}...")
|
|
52
|
+
print(f" URL: {url}")
|
|
53
|
+
print(f" → {output_path}")
|
|
54
|
+
|
|
55
|
+
try:
|
|
56
|
+
# Create a custom progress hook
|
|
57
|
+
def reporthook(blocknum, blocksize, totalsize):
|
|
58
|
+
if totalsize > 0:
|
|
59
|
+
percent = min(blocknum * blocksize / totalsize * 100, 100)
|
|
60
|
+
mb_downloaded = blocknum * blocksize / 1024 / 1024
|
|
61
|
+
mb_total = totalsize / 1024 / 1024
|
|
62
|
+
print(f"\r Progress: {percent:.1f}% ({mb_downloaded:.1f}/{mb_total:.1f} MB)", end='')
|
|
63
|
+
|
|
64
|
+
urllib.request.urlretrieve(url, output_path, reporthook=reporthook)
|
|
65
|
+
print() # New line after progress
|
|
66
|
+
print(f" ✓ Downloaded successfully")
|
|
67
|
+
except Exception as e:
|
|
68
|
+
print(f"\n ✗ Download failed: {e}")
|
|
69
|
+
raise
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _decompress_gz(gz_path: Path, output_path: Path) -> None:
|
|
73
|
+
"""
|
|
74
|
+
Decompress a gzipped file.
|
|
75
|
+
|
|
76
|
+
Parameters
|
|
77
|
+
----------
|
|
78
|
+
gz_path : Path
|
|
79
|
+
Path to .gz file
|
|
80
|
+
output_path : Path
|
|
81
|
+
Path for decompressed output
|
|
82
|
+
"""
|
|
83
|
+
print(f"📦 Decompressing {gz_path.name}...")
|
|
84
|
+
with gzip.open(gz_path, 'rb') as f_in:
|
|
85
|
+
with open(output_path, 'wb') as f_out:
|
|
86
|
+
shutil.copyfileobj(f_in, f_out)
|
|
87
|
+
print(f" ✓ Decompressed to {output_path.name}")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def mouse_brain_5k_atacseq(force_download: bool = False, use_local: bool = True) -> Tuple[Path, Path]:
|
|
91
|
+
"""
|
|
92
|
+
Download 10X Mouse Brain 5k scATAC-seq dataset and mouse annotation.
|
|
93
|
+
|
|
94
|
+
This dataset contains ~5,000 cells from mouse brain tissue, profiled
|
|
95
|
+
with 10X Chromium scATAC-seq. Automatically downloads both the peak
|
|
96
|
+
matrix (H5 format) and the mouse GENCODE annotation (GTF).
|
|
97
|
+
|
|
98
|
+
Parameters
|
|
99
|
+
----------
|
|
100
|
+
force_download : bool, optional
|
|
101
|
+
If True, re-download even if files exist (default: False)
|
|
102
|
+
use_local : bool, optional
|
|
103
|
+
If True, check examples/data/ for files before downloading (default: True)
|
|
104
|
+
|
|
105
|
+
Returns
|
|
106
|
+
-------
|
|
107
|
+
h5_path : Path
|
|
108
|
+
Path to filtered_peak_bc_matrix.h5 file
|
|
109
|
+
gtf_path : Path
|
|
110
|
+
Path to gencode.vM25.annotation.gtf file
|
|
111
|
+
|
|
112
|
+
Examples
|
|
113
|
+
--------
|
|
114
|
+
>>> import iaode
|
|
115
|
+
>>> h5_file, gtf_file = iaode.datasets.mouse_brain_5k_atacseq()
|
|
116
|
+
>>> adata = iaode.annotation_pipeline(h5_file, gtf_file)
|
|
117
|
+
|
|
118
|
+
Notes
|
|
119
|
+
-----
|
|
120
|
+
If automatic download fails (403 error), the function will look for files in:
|
|
121
|
+
- Current directory: examples/data/
|
|
122
|
+
- User cache: ~/.iaode/data/
|
|
123
|
+
|
|
124
|
+
Manual download instructions will be provided if files are not found.
|
|
125
|
+
"""
|
|
126
|
+
data_dir = get_data_dir()
|
|
127
|
+
atacseq_dir = data_dir / "atacseq"
|
|
128
|
+
anno_dir = data_dir / "annotations"
|
|
129
|
+
|
|
130
|
+
# Define file paths
|
|
131
|
+
h5_file = atacseq_dir / "mouse_brain_5k_v1.1.h5"
|
|
132
|
+
gtf_file = anno_dir / "gencode.vM25.annotation.gtf"
|
|
133
|
+
gtf_gz = anno_dir / "gencode.vM25.annotation.gtf.gz"
|
|
134
|
+
|
|
135
|
+
# Check for local files first (e.g., in examples/data/)
|
|
136
|
+
if use_local:
|
|
137
|
+
local_examples_dir = Path.cwd() / "data"
|
|
138
|
+
if not local_examples_dir.exists():
|
|
139
|
+
local_examples_dir = Path.cwd().parent / "data" # Try parent dir
|
|
140
|
+
|
|
141
|
+
local_h5 = local_examples_dir / "mouse_brain_5k_v1.1.h5"
|
|
142
|
+
local_gtf = local_examples_dir / "gencode.vM25.annotation.gtf"
|
|
143
|
+
|
|
144
|
+
if local_h5.exists() and local_gtf.exists():
|
|
145
|
+
print(f"✓ Using local files from: {local_examples_dir}")
|
|
146
|
+
# Copy to cache for consistency
|
|
147
|
+
if not h5_file.exists():
|
|
148
|
+
print(f" Copying H5 to cache...")
|
|
149
|
+
shutil.copy(local_h5, h5_file)
|
|
150
|
+
if not gtf_file.exists():
|
|
151
|
+
print(f" Copying GTF to cache...")
|
|
152
|
+
shutil.copy(local_gtf, gtf_file)
|
|
153
|
+
print()
|
|
154
|
+
print("=" * 70)
|
|
155
|
+
print("✅ Dataset ready!")
|
|
156
|
+
print("=" * 70)
|
|
157
|
+
print(f"H5 file: {h5_file}")
|
|
158
|
+
print(f"GTF file: {gtf_file}")
|
|
159
|
+
print(f"Cache location: {data_dir}")
|
|
160
|
+
print("=" * 70)
|
|
161
|
+
print()
|
|
162
|
+
return h5_file, gtf_file
|
|
163
|
+
|
|
164
|
+
# Download H5 file if needed
|
|
165
|
+
if not h5_file.exists() or force_download:
|
|
166
|
+
h5_url = "https://cf.10xgenomics.com/samples/cell-atac/2.0.0/atac_mouse_brain_5k_v1.1/atac_mouse_brain_5k_v1.1_filtered_peak_bc_matrix.h5"
|
|
167
|
+
try:
|
|
168
|
+
_download_file(h5_url, h5_file, "mouse brain 5k scATAC-seq H5 matrix")
|
|
169
|
+
except Exception as e:
|
|
170
|
+
print(f"\n⚠️ Automatic download failed: {e}")
|
|
171
|
+
print("\n" + "=" * 70)
|
|
172
|
+
print("MANUAL DOWNLOAD REQUIRED")
|
|
173
|
+
print("=" * 70)
|
|
174
|
+
print("Please download the file manually:")
|
|
175
|
+
print(f"\n1. Visit: https://www.10xgenomics.com/datasets/fresh-cortex-from-adult-mouse-brain-p-50-1-standard-2-0-0")
|
|
176
|
+
print(" Or try direct link:")
|
|
177
|
+
print(f" {h5_url}")
|
|
178
|
+
print(f"\n2. Save as: {h5_file}")
|
|
179
|
+
print(f"\n3. Or place in: {Path.cwd() / 'data' / 'mouse_brain_5k_v1.1.h5'}")
|
|
180
|
+
print("=" * 70)
|
|
181
|
+
raise
|
|
182
|
+
else:
|
|
183
|
+
print(f"✓ Using cached H5: {h5_file}")
|
|
184
|
+
|
|
185
|
+
# Download GTF if needed
|
|
186
|
+
if not gtf_file.exists() or force_download:
|
|
187
|
+
if not gtf_gz.exists() or force_download:
|
|
188
|
+
gtf_url = "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M25/gencode.vM25.annotation.gtf.gz"
|
|
189
|
+
_download_file(gtf_url, gtf_gz, "GENCODE vM25 mouse annotation (847 MB compressed)")
|
|
190
|
+
|
|
191
|
+
# Decompress
|
|
192
|
+
_decompress_gz(gtf_gz, gtf_file)
|
|
193
|
+
|
|
194
|
+
# Clean up compressed file to save space
|
|
195
|
+
if gtf_gz.exists():
|
|
196
|
+
gtf_gz.unlink()
|
|
197
|
+
print(f" ✓ Removed compressed file to save space")
|
|
198
|
+
else:
|
|
199
|
+
print(f"✓ Using cached GTF: {gtf_file}")
|
|
200
|
+
|
|
201
|
+
print()
|
|
202
|
+
print("=" * 70)
|
|
203
|
+
print("✅ Dataset ready!")
|
|
204
|
+
print("=" * 70)
|
|
205
|
+
print(f"H5 file: {h5_file}")
|
|
206
|
+
print(f"GTF file: {gtf_file}")
|
|
207
|
+
print(f"Cache location: {data_dir}")
|
|
208
|
+
print("=" * 70)
|
|
209
|
+
print()
|
|
210
|
+
|
|
211
|
+
return h5_file, gtf_file
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def human_pbmc_5k_atacseq(force_download: bool = False) -> Tuple[Path, Path]:
|
|
215
|
+
"""
|
|
216
|
+
Download 10X Human PBMC 5k scATAC-seq dataset and human annotation.
|
|
217
|
+
|
|
218
|
+
This dataset contains ~5,000 peripheral blood mononuclear cells (PBMCs),
|
|
219
|
+
profiled with 10X Chromium scATAC-seq. Automatically downloads both the
|
|
220
|
+
peak matrix (H5 format) and the human GENCODE annotation (GTF).
|
|
221
|
+
|
|
222
|
+
Parameters
|
|
223
|
+
----------
|
|
224
|
+
force_download : bool, optional
|
|
225
|
+
If True, re-download even if files exist (default: False)
|
|
226
|
+
|
|
227
|
+
Returns
|
|
228
|
+
-------
|
|
229
|
+
h5_path : Path
|
|
230
|
+
Path to filtered_peak_bc_matrix.h5 file
|
|
231
|
+
gtf_path : Path
|
|
232
|
+
Path to gencode.v49.annotation.gtf file
|
|
233
|
+
|
|
234
|
+
Examples
|
|
235
|
+
--------
|
|
236
|
+
>>> import iaode
|
|
237
|
+
>>> h5_file, gtf_file = iaode.datasets.human_pbmc_5k_atacseq()
|
|
238
|
+
>>> adata = iaode.annotation_pipeline(h5_file, gtf_file)
|
|
239
|
+
"""
|
|
240
|
+
data_dir = get_data_dir()
|
|
241
|
+
atacseq_dir = data_dir / "atacseq"
|
|
242
|
+
anno_dir = data_dir / "annotations"
|
|
243
|
+
|
|
244
|
+
# Define file paths
|
|
245
|
+
h5_file = atacseq_dir / "human_pbmc_5k_nextgem.h5"
|
|
246
|
+
gtf_file = anno_dir / "gencode.v49.annotation.gtf"
|
|
247
|
+
gtf_gz = anno_dir / "gencode.v49.annotation.gtf.gz"
|
|
248
|
+
|
|
249
|
+
# Download H5 file if needed
|
|
250
|
+
if not h5_file.exists() or force_download:
|
|
251
|
+
h5_url = "https://cf.10xgenomics.com/samples/cell-atac/2.0.0/atac_pbmc_5k_nextgem/atac_pbmc_5k_nextgem_filtered_peak_bc_matrix.h5"
|
|
252
|
+
_download_file(h5_url, h5_file, "human PBMC 5k scATAC-seq H5 matrix")
|
|
253
|
+
else:
|
|
254
|
+
print(f"✓ Using cached H5: {h5_file}")
|
|
255
|
+
|
|
256
|
+
# Download GTF if needed
|
|
257
|
+
if not gtf_file.exists() or force_download:
|
|
258
|
+
if not gtf_gz.exists() or force_download:
|
|
259
|
+
gtf_url = "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_49/gencode.v49.annotation.gtf.gz"
|
|
260
|
+
_download_file(gtf_url, gtf_gz, "GENCODE v49 human annotation (1.4 GB compressed)")
|
|
261
|
+
|
|
262
|
+
# Decompress
|
|
263
|
+
_decompress_gz(gtf_gz, gtf_file)
|
|
264
|
+
|
|
265
|
+
# Clean up compressed file
|
|
266
|
+
if gtf_gz.exists():
|
|
267
|
+
gtf_gz.unlink()
|
|
268
|
+
print(f" ✓ Removed compressed file to save space")
|
|
269
|
+
else:
|
|
270
|
+
print(f"✓ Using cached GTF: {gtf_file}")
|
|
271
|
+
|
|
272
|
+
print()
|
|
273
|
+
print("=" * 70)
|
|
274
|
+
print("✅ Dataset ready!")
|
|
275
|
+
print("=" * 70)
|
|
276
|
+
print(f"H5 file: {h5_file}")
|
|
277
|
+
print(f"GTF file: {gtf_file}")
|
|
278
|
+
print(f"Cache location: {data_dir}")
|
|
279
|
+
print("=" * 70)
|
|
280
|
+
print()
|
|
281
|
+
|
|
282
|
+
return h5_file, gtf_file
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def clear_cache() -> None:
|
|
286
|
+
"""
|
|
287
|
+
Clear the iAODE data cache directory.
|
|
288
|
+
|
|
289
|
+
Removes all downloaded datasets to free disk space.
|
|
290
|
+
Files will be re-downloaded on next use.
|
|
291
|
+
"""
|
|
292
|
+
data_dir = get_data_dir()
|
|
293
|
+
if data_dir.exists():
|
|
294
|
+
shutil.rmtree(data_dir)
|
|
295
|
+
print(f"✓ Cleared cache: {data_dir}")
|
|
296
|
+
print(" Files will be re-downloaded when needed.")
|
|
297
|
+
else:
|
|
298
|
+
print("✓ Cache already empty")
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def list_cached_files() -> None:
|
|
302
|
+
"""
|
|
303
|
+
List all cached dataset files and their sizes.
|
|
304
|
+
"""
|
|
305
|
+
data_dir = get_data_dir()
|
|
306
|
+
print("=" * 70)
|
|
307
|
+
print("iAODE Cached Datasets")
|
|
308
|
+
print("=" * 70)
|
|
309
|
+
print(f"Location: {data_dir}")
|
|
310
|
+
print()
|
|
311
|
+
|
|
312
|
+
total_size = 0
|
|
313
|
+
file_count = 0
|
|
314
|
+
|
|
315
|
+
for root, dirs, files in os.walk(data_dir):
|
|
316
|
+
for file in files:
|
|
317
|
+
filepath = Path(root) / file
|
|
318
|
+
size_mb = filepath.stat().st_size / 1024 / 1024
|
|
319
|
+
total_size += size_mb
|
|
320
|
+
file_count += 1
|
|
321
|
+
rel_path = filepath.relative_to(data_dir)
|
|
322
|
+
print(f" {rel_path}")
|
|
323
|
+
print(f" Size: {size_mb:.1f} MB")
|
|
324
|
+
|
|
325
|
+
print()
|
|
326
|
+
print(f"Total: {file_count} files, {total_size:.1f} MB")
|
|
327
|
+
print("=" * 70)
|
|
328
|
+
|
|
329
|
+
if file_count > 0:
|
|
330
|
+
print("\nTo clear cache: iaode.datasets.clear_cache()")
|
iaode/environment.py
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
# environment.py
|
|
2
|
+
|
|
3
|
+
from .model import iaodeModel
|
|
4
|
+
from .mixin import envMixin
|
|
5
|
+
import numpy as np
|
|
6
|
+
from sklearn.cluster import KMeans
|
|
7
|
+
from sklearn.preprocessing import LabelEncoder
|
|
8
|
+
import torch
|
|
9
|
+
from torch.utils.data import DataLoader, TensorDataset
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Env(iaodeModel, envMixin):
|
|
13
|
+
"""
|
|
14
|
+
Training environment for iAODE model.
|
|
15
|
+
|
|
16
|
+
Handles data registration, train/val/test splitting, DataLoader creation,
|
|
17
|
+
epoch-based training, validation, and early stopping.
|
|
18
|
+
|
|
19
|
+
Parameters
|
|
20
|
+
----------
|
|
21
|
+
adata : AnnData
|
|
22
|
+
Annotated data object
|
|
23
|
+
layer : str
|
|
24
|
+
Layer name for count data
|
|
25
|
+
train_size : float
|
|
26
|
+
Training set fraction
|
|
27
|
+
val_size : float
|
|
28
|
+
Validation set fraction
|
|
29
|
+
test_size : float
|
|
30
|
+
Test set fraction
|
|
31
|
+
batch_size : int
|
|
32
|
+
Batch size for training
|
|
33
|
+
random_seed : int
|
|
34
|
+
Random seed for reproducibility
|
|
35
|
+
**kwargs : dict
|
|
36
|
+
Additional parameters passed to iaodeModel
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
adata,
|
|
42
|
+
layer,
|
|
43
|
+
recon,
|
|
44
|
+
irecon,
|
|
45
|
+
beta,
|
|
46
|
+
dip,
|
|
47
|
+
tc,
|
|
48
|
+
info,
|
|
49
|
+
hidden_dim,
|
|
50
|
+
latent_dim,
|
|
51
|
+
i_dim,
|
|
52
|
+
use_ode,
|
|
53
|
+
loss_mode,
|
|
54
|
+
lr,
|
|
55
|
+
vae_reg,
|
|
56
|
+
ode_reg,
|
|
57
|
+
device,
|
|
58
|
+
train_size,
|
|
59
|
+
val_size,
|
|
60
|
+
test_size,
|
|
61
|
+
batch_size,
|
|
62
|
+
random_seed,
|
|
63
|
+
encoder_type,
|
|
64
|
+
encoder_num_layers,
|
|
65
|
+
encoder_n_heads,
|
|
66
|
+
encoder_d_model,
|
|
67
|
+
*args,
|
|
68
|
+
**kwargs,
|
|
69
|
+
):
|
|
70
|
+
# Store split parameters
|
|
71
|
+
self.train_size = train_size
|
|
72
|
+
self.val_size = val_size
|
|
73
|
+
self.test_size = test_size
|
|
74
|
+
self.batch_size_fixed = batch_size
|
|
75
|
+
self.random_seed = random_seed
|
|
76
|
+
|
|
77
|
+
# Register data and create splits
|
|
78
|
+
self._register_anndata(adata, layer, latent_dim)
|
|
79
|
+
|
|
80
|
+
# Initialize model
|
|
81
|
+
super().__init__(
|
|
82
|
+
recon=recon,
|
|
83
|
+
irecon=irecon,
|
|
84
|
+
beta=beta,
|
|
85
|
+
dip=dip,
|
|
86
|
+
tc=tc,
|
|
87
|
+
info=info,
|
|
88
|
+
state_dim=self.n_var,
|
|
89
|
+
hidden_dim=hidden_dim,
|
|
90
|
+
latent_dim=latent_dim,
|
|
91
|
+
i_dim=i_dim,
|
|
92
|
+
use_ode=use_ode,
|
|
93
|
+
loss_mode=loss_mode,
|
|
94
|
+
lr=lr,
|
|
95
|
+
vae_reg=vae_reg,
|
|
96
|
+
ode_reg=ode_reg,
|
|
97
|
+
device=device,
|
|
98
|
+
encoder_type=encoder_type,
|
|
99
|
+
encoder_num_layers=encoder_num_layers,
|
|
100
|
+
encoder_n_heads=encoder_n_heads,
|
|
101
|
+
encoder_d_model=encoder_d_model,
|
|
102
|
+
*args,
|
|
103
|
+
**kwargs,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
# Training tracking
|
|
107
|
+
self.score = []
|
|
108
|
+
self.train_losses = []
|
|
109
|
+
self.val_losses = []
|
|
110
|
+
self.val_scores = []
|
|
111
|
+
|
|
112
|
+
# Early stopping state
|
|
113
|
+
self.best_val_loss = float('inf')
|
|
114
|
+
self.best_model_state = None
|
|
115
|
+
self.patience_counter = 0
|
|
116
|
+
|
|
117
|
+
def _register_anndata(self, adata, layer: str, latent_dim: int):
|
|
118
|
+
"""Load data and create train/val/test splits"""
|
|
119
|
+
|
|
120
|
+
# Load and log-transform data
|
|
121
|
+
if hasattr(adata.layers[layer], 'toarray'):
|
|
122
|
+
self.X = np.log1p(adata.layers[layer].toarray())
|
|
123
|
+
else:
|
|
124
|
+
self.X = np.log1p(adata.layers[layer])
|
|
125
|
+
|
|
126
|
+
self.n_obs = adata.shape[0]
|
|
127
|
+
self.n_var = adata.shape[1]
|
|
128
|
+
|
|
129
|
+
# Generate evaluation labels
|
|
130
|
+
if 'cell_type' in adata.obs.columns:
|
|
131
|
+
self.labels = LabelEncoder().fit_transform(adata.obs['cell_type'])
|
|
132
|
+
else:
|
|
133
|
+
# Cast to ndarray to ensure downstream indexing works without type warnings
|
|
134
|
+
self.labels = np.asarray(
|
|
135
|
+
KMeans(latent_dim, random_state=self.random_seed).fit_predict(self.X)
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# Create train/val/test splits
|
|
139
|
+
np.random.seed(self.random_seed)
|
|
140
|
+
indices = np.random.permutation(self.n_obs)
|
|
141
|
+
|
|
142
|
+
n_train = int(self.train_size * self.n_obs)
|
|
143
|
+
n_val = int(self.val_size * self.n_obs)
|
|
144
|
+
|
|
145
|
+
self.train_idx = indices[:n_train]
|
|
146
|
+
self.val_idx = indices[n_train:n_train + n_val]
|
|
147
|
+
self.test_idx = indices[n_train + n_val:]
|
|
148
|
+
|
|
149
|
+
# Split data
|
|
150
|
+
self.X_train = self.X[self.train_idx]
|
|
151
|
+
self.X_val = self.X[self.val_idx]
|
|
152
|
+
self.X_test = self.X[self.test_idx]
|
|
153
|
+
|
|
154
|
+
# Indexing warnings suppressed – runtime types are ndarray
|
|
155
|
+
self.labels_train = self.labels[self.train_idx] # type: ignore[index]
|
|
156
|
+
self.labels_val = self.labels[self.val_idx] # type: ignore[index]
|
|
157
|
+
self.labels_test = self.labels[self.test_idx] # type: ignore[index]
|
|
158
|
+
|
|
159
|
+
print("\n" + "="*70)
|
|
160
|
+
print("Data Split")
|
|
161
|
+
print("="*70)
|
|
162
|
+
print(f" Train: {len(self.train_idx):,} cells ({len(self.train_idx)/self.n_obs*100:.1f}%)")
|
|
163
|
+
print(f" Val: {len(self.val_idx):,} cells ({len(self.val_idx)/self.n_obs*100:.1f}%)")
|
|
164
|
+
print(f" Test: {len(self.test_idx):,} cells ({len(self.test_idx)/self.n_obs*100:.1f}%)")
|
|
165
|
+
|
|
166
|
+
# Create DataLoaders
|
|
167
|
+
self._create_dataloaders()
|
|
168
|
+
|
|
169
|
+
def _create_dataloaders(self):
|
|
170
|
+
"""Create PyTorch DataLoaders for training, validation, and testing"""
|
|
171
|
+
|
|
172
|
+
# Convert to tensors
|
|
173
|
+
X_train_tensor = torch.FloatTensor(self.X_train)
|
|
174
|
+
X_val_tensor = torch.FloatTensor(self.X_val)
|
|
175
|
+
X_test_tensor = torch.FloatTensor(self.X_test)
|
|
176
|
+
|
|
177
|
+
# Create datasets
|
|
178
|
+
train_dataset = TensorDataset(X_train_tensor)
|
|
179
|
+
val_dataset = TensorDataset(X_val_tensor)
|
|
180
|
+
test_dataset = TensorDataset(X_test_tensor)
|
|
181
|
+
|
|
182
|
+
# Create dataloaders
|
|
183
|
+
self.train_loader = DataLoader(
|
|
184
|
+
train_dataset,
|
|
185
|
+
batch_size=self.batch_size_fixed,
|
|
186
|
+
shuffle=True,
|
|
187
|
+
drop_last=True
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
self.val_loader = DataLoader(
|
|
191
|
+
val_dataset,
|
|
192
|
+
batch_size=self.batch_size_fixed,
|
|
193
|
+
shuffle=False,
|
|
194
|
+
drop_last=False
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
self.test_loader = DataLoader(
|
|
198
|
+
test_dataset,
|
|
199
|
+
batch_size=self.batch_size_fixed,
|
|
200
|
+
shuffle=False,
|
|
201
|
+
drop_last=False
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
print(f" Batches per epoch: {len(self.train_loader)}")
|
|
205
|
+
print("="*70)
|
|
206
|
+
|
|
207
|
+
def train_epoch(self):
|
|
208
|
+
"""Train for one complete epoch"""
|
|
209
|
+
|
|
210
|
+
self.train()
|
|
211
|
+
epoch_losses = []
|
|
212
|
+
|
|
213
|
+
for batch_data, in self.train_loader:
|
|
214
|
+
batch_data = batch_data.to(self.device)
|
|
215
|
+
self.update(batch_data)
|
|
216
|
+
epoch_losses.append(self.loss[-1][0])
|
|
217
|
+
|
|
218
|
+
avg_train_loss = np.mean(epoch_losses)
|
|
219
|
+
self.train_losses.append(avg_train_loss)
|
|
220
|
+
|
|
221
|
+
return avg_train_loss
|
|
222
|
+
|
|
223
|
+
def validate(self):
|
|
224
|
+
"""Evaluate on validation set"""
|
|
225
|
+
|
|
226
|
+
self.eval()
|
|
227
|
+
val_losses = []
|
|
228
|
+
all_latents = []
|
|
229
|
+
|
|
230
|
+
with torch.no_grad():
|
|
231
|
+
for batch_data, in self.val_loader:
|
|
232
|
+
batch_data = batch_data.to(self.device)
|
|
233
|
+
|
|
234
|
+
# Compute loss
|
|
235
|
+
loss_value = self._compute_loss_only(batch_data)
|
|
236
|
+
val_losses.append(loss_value)
|
|
237
|
+
|
|
238
|
+
# Extract latent representations
|
|
239
|
+
latent = self.take_latent(batch_data)
|
|
240
|
+
all_latents.append(latent)
|
|
241
|
+
|
|
242
|
+
# Average validation loss
|
|
243
|
+
avg_val_loss = np.mean(val_losses)
|
|
244
|
+
self.val_losses.append(avg_val_loss)
|
|
245
|
+
|
|
246
|
+
# Compute clustering metrics
|
|
247
|
+
all_latents = np.concatenate(all_latents, axis=0)
|
|
248
|
+
val_score = self._calc_score_with_labels(all_latents, self.labels_val)
|
|
249
|
+
self.val_scores.append(val_score)
|
|
250
|
+
|
|
251
|
+
return avg_val_loss, val_score
|
|
252
|
+
|
|
253
|
+
def check_early_stopping(self, val_loss: float, patience: int = 20):
|
|
254
|
+
"""
|
|
255
|
+
Check early stopping criterion.
|
|
256
|
+
|
|
257
|
+
Parameters
|
|
258
|
+
----------
|
|
259
|
+
val_loss : float
|
|
260
|
+
Current validation loss
|
|
261
|
+
patience : int, default=20
|
|
262
|
+
Number of epochs without improvement before stopping
|
|
263
|
+
|
|
264
|
+
Returns
|
|
265
|
+
-------
|
|
266
|
+
should_stop : bool
|
|
267
|
+
Whether training should stop
|
|
268
|
+
improved : bool
|
|
269
|
+
Whether validation loss improved
|
|
270
|
+
"""
|
|
271
|
+
|
|
272
|
+
if val_loss < self.best_val_loss:
|
|
273
|
+
# Save best model
|
|
274
|
+
self.best_val_loss = val_loss
|
|
275
|
+
self.best_model_state = {
|
|
276
|
+
k: v.cpu().clone() for k, v in self.state_dict().items()
|
|
277
|
+
}
|
|
278
|
+
self.patience_counter = 0
|
|
279
|
+
return False, True
|
|
280
|
+
else:
|
|
281
|
+
# Increment patience counter
|
|
282
|
+
self.patience_counter += 1
|
|
283
|
+
|
|
284
|
+
if self.patience_counter >= patience:
|
|
285
|
+
return True, False
|
|
286
|
+
else:
|
|
287
|
+
return False, False
|
|
288
|
+
|
|
289
|
+
def load_best_model(self):
|
|
290
|
+
"""Restore best model from early stopping checkpoint"""
|
|
291
|
+
|
|
292
|
+
if self.best_model_state is not None:
|
|
293
|
+
self.load_state_dict(self.best_model_state)
|
|
294
|
+
print(f"\n{'='*70}")
|
|
295
|
+
print(f"Loaded best model (val_loss={self.best_val_loss:.4f})")
|
|
296
|
+
print(f"{'='*70}\n")
|
|
297
|
+
else:
|
|
298
|
+
print("\nWarning: No saved model state found!")
|