iaode 0.2.7__tar.gz → 0.2.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {iaode-0.2.7/iaode.egg-info → iaode-0.2.9}/PKG-INFO +21 -15
- {iaode-0.2.7 → iaode-0.2.9}/README.md +20 -14
- {iaode-0.2.7 → iaode-0.2.9}/iaode/__init__.py +1 -1
- {iaode-0.2.7 → iaode-0.2.9}/iaode/agent.py +95 -62
- iaode-0.2.9/iaode/datasets.py +294 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode/utils.py +94 -15
- {iaode-0.2.7 → iaode-0.2.9/iaode.egg-info}/PKG-INFO +21 -15
- {iaode-0.2.7 → iaode-0.2.9}/pyproject.toml +1 -1
- iaode-0.2.7/iaode/datasets.py +0 -329
- {iaode-0.2.7 → iaode-0.2.9}/LICENSE +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/MANIFEST.in +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode/BEN.py +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode/DRE.py +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode/LSE.py +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode/annotation.py +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode/environment.py +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode/mixin.py +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode/model.py +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode/module.py +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode.egg-info/SOURCES.txt +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode.egg-info/dependency_links.txt +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode.egg-info/requires.txt +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/iaode.egg-info/top_level.txt +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/requirements.txt +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/setup.cfg +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/setup.py +0 -0
- {iaode-0.2.7 → iaode-0.2.9}/tests/test_smoke.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: iaode
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.9
|
|
4
4
|
Summary: Interpretable Autoencoder with Ordinary Differential Equations for single-cell omics analysis
|
|
5
5
|
Author-email: Zeyu Fu <fuzeyu99@126.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -98,34 +98,40 @@ Loss = recon + i_recon + ODE_consistency + β·KL + dip·DIP + tc·TC + info·MM
|
|
|
98
98
|
|
|
99
99
|
## Installation
|
|
100
100
|
|
|
101
|
-
###
|
|
102
|
-
|
|
101
|
+
### Method 1: Python Library (Standard)
|
|
102
|
+
For users using iAODE in scripts or notebooks:
|
|
103
103
|
```bash
|
|
104
104
|
pip install iaode
|
|
105
105
|
```
|
|
106
106
|
|
|
107
|
-
###
|
|
107
|
+
### Method 2: Web Interface (Full)
|
|
108
|
+
For users who want the **Interactive Training UI**, you must clone the repository (since the UI requires the server files):
|
|
108
109
|
|
|
109
110
|
```bash
|
|
110
111
|
git clone https://github.com/PeterPonyu/iAODE.git
|
|
111
112
|
cd iAODE
|
|
112
113
|
pip install -e .
|
|
114
|
+
pip install fastapi uvicorn # Required for the UI server
|
|
113
115
|
```
|
|
114
116
|
|
|
115
|
-
### Requirements
|
|
116
|
-
|
|
117
|
-
- Python ≥ 3.9
|
|
118
|
-
- PyTorch ≥ 1.10.0
|
|
119
|
-
- AnnData ≥ 0.8.0
|
|
120
|
-
- Scanpy ≥ 1.8.0
|
|
121
|
-
- scvi-tools ≥ 0.16.0
|
|
122
|
-
|
|
123
|
-
See `requirements.txt` for the complete list of dependencies.
|
|
124
|
-
|
|
125
117
|
---
|
|
126
|
-
|
|
127
118
|
## Quick Start
|
|
128
119
|
|
|
120
|
+
### 🖥️ Interactive Web Interface
|
|
121
|
+
**No Node.js required.** The interface is pre-built and served via Python.
|
|
122
|
+
|
|
123
|
+
1. **Start the Server:**
|
|
124
|
+
```bash
|
|
125
|
+
python api/run_server.py
|
|
126
|
+
```
|
|
127
|
+
2. **Access the UI:**
|
|
128
|
+
Open **[http://localhost:8000/](http://localhost:8000/)** in your browser.
|
|
129
|
+
|
|
130
|
+
*Features:*
|
|
131
|
+
* Upload `.h5ad` or `.h5` files.
|
|
132
|
+
* Configure Encoders, Loss functions, and ODE parameters visually.
|
|
133
|
+
* Download trained embeddings directly.
|
|
134
|
+
|
|
129
135
|
### Basic scATAC-seq Workflow
|
|
130
136
|
|
|
131
137
|
```python
|
|
@@ -49,34 +49,40 @@ Loss = recon + i_recon + ODE_consistency + β·KL + dip·DIP + tc·TC + info·MM
|
|
|
49
49
|
|
|
50
50
|
## Installation
|
|
51
51
|
|
|
52
|
-
###
|
|
53
|
-
|
|
52
|
+
### Method 1: Python Library (Standard)
|
|
53
|
+
For users using iAODE in scripts or notebooks:
|
|
54
54
|
```bash
|
|
55
55
|
pip install iaode
|
|
56
56
|
```
|
|
57
57
|
|
|
58
|
-
###
|
|
58
|
+
### Method 2: Web Interface (Full)
|
|
59
|
+
For users who want the **Interactive Training UI**, you must clone the repository (since the UI requires the server files):
|
|
59
60
|
|
|
60
61
|
```bash
|
|
61
62
|
git clone https://github.com/PeterPonyu/iAODE.git
|
|
62
63
|
cd iAODE
|
|
63
64
|
pip install -e .
|
|
65
|
+
pip install fastapi uvicorn # Required for the UI server
|
|
64
66
|
```
|
|
65
67
|
|
|
66
|
-
### Requirements
|
|
67
|
-
|
|
68
|
-
- Python ≥ 3.9
|
|
69
|
-
- PyTorch ≥ 1.10.0
|
|
70
|
-
- AnnData ≥ 0.8.0
|
|
71
|
-
- Scanpy ≥ 1.8.0
|
|
72
|
-
- scvi-tools ≥ 0.16.0
|
|
73
|
-
|
|
74
|
-
See `requirements.txt` for the complete list of dependencies.
|
|
75
|
-
|
|
76
68
|
---
|
|
77
|
-
|
|
78
69
|
## Quick Start
|
|
79
70
|
|
|
71
|
+
### 🖥️ Interactive Web Interface
|
|
72
|
+
**No Node.js required.** The interface is pre-built and served via Python.
|
|
73
|
+
|
|
74
|
+
1. **Start the Server:**
|
|
75
|
+
```bash
|
|
76
|
+
python api/run_server.py
|
|
77
|
+
```
|
|
78
|
+
2. **Access the UI:**
|
|
79
|
+
Open **[http://localhost:8000/](http://localhost:8000/)** in your browser.
|
|
80
|
+
|
|
81
|
+
*Features:*
|
|
82
|
+
* Upload `.h5ad` or `.h5` files.
|
|
83
|
+
* Configure Encoders, Loss functions, and ODE parameters visually.
|
|
84
|
+
* Download trained embeddings directly.
|
|
85
|
+
|
|
80
86
|
### Basic scATAC-seq Workflow
|
|
81
87
|
|
|
82
88
|
```python
|
|
@@ -5,7 +5,7 @@ import scanpy as sc # type: ignore
|
|
|
5
5
|
from anndata import AnnData # type: ignore
|
|
6
6
|
import numpy as np
|
|
7
7
|
import torch
|
|
8
|
-
import tqdm
|
|
8
|
+
from tqdm.auto import tqdm # type: ignore
|
|
9
9
|
import time
|
|
10
10
|
from typing import Literal, Optional
|
|
11
11
|
from scipy.stats import norm # type: ignore
|
|
@@ -183,80 +183,113 @@ class agent(Env):
|
|
|
183
183
|
val_every: int = 5,
|
|
184
184
|
early_stop: bool = True,
|
|
185
185
|
):
|
|
186
|
-
"""Train model with early stopping."""
|
|
186
|
+
"""Train model with early stopping and clean progress tracking."""
|
|
187
187
|
|
|
188
|
-
# Initialize resource tracking
|
|
189
188
|
use_cuda = torch.cuda.is_available()
|
|
190
189
|
if use_cuda:
|
|
191
190
|
torch.cuda.reset_peak_memory_stats()
|
|
192
191
|
start_time = time.time()
|
|
193
192
|
|
|
194
|
-
#
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
193
|
+
# Prepare header & separator (but do NOT print yet)
|
|
194
|
+
header = (
|
|
195
|
+
f"{'Epoch':<6} | "
|
|
196
|
+
f"{'Train Loss':<10} | "
|
|
197
|
+
f"{'Val Loss':<10} | "
|
|
198
|
+
f"{'ARI':<6} {'NMI':<6} {'ASW':<6} | "
|
|
199
|
+
f"{'CAL':<6} {'DAV':<6} {'COR':<6} | "
|
|
200
|
+
f"{'Patience':<9}"
|
|
201
|
+
)
|
|
202
|
+
sep_line = "-" * len(header)
|
|
203
|
+
|
|
204
|
+
# Single unified progress bar
|
|
205
|
+
with tqdm(total=epochs, desc="Training", unit="epoch", dynamic_ncols=True) as pbar:
|
|
206
|
+
# Print header via pbar.write so that header & rows stay together
|
|
207
|
+
pbar.write(sep_line)
|
|
208
|
+
pbar.write(header)
|
|
209
|
+
pbar.write(sep_line)
|
|
210
|
+
|
|
211
|
+
for epoch in range(epochs):
|
|
212
|
+
|
|
213
|
+
# 1. Training step
|
|
214
|
+
train_loss = self.train_epoch()
|
|
215
|
+
|
|
216
|
+
# 2. Validation step
|
|
217
|
+
if (epoch + 1) % val_every == 0 or epoch == 0:
|
|
218
|
+
val_loss, val_score = self.validate()
|
|
219
|
+
val_loss = float(val_loss)
|
|
220
|
+
|
|
221
|
+
# val_score indices:
|
|
222
|
+
# 0: ARI, 1: NMI, 2: ASW, 3: CAL, 4: DAV, 5: COR
|
|
223
|
+
ari, nmi, asw, cal, dav, cor = val_score[:6]
|
|
224
|
+
|
|
225
|
+
if early_stop:
|
|
226
|
+
should_stop, _ = self.check_early_stopping(val_loss, patience)
|
|
227
|
+
pat_status = f"{self.patience_counter}/{patience}"
|
|
228
|
+
else:
|
|
229
|
+
should_stop = False
|
|
230
|
+
pat_status = "N/A"
|
|
231
|
+
|
|
232
|
+
# Log line with all 6 metrics
|
|
233
|
+
log_msg = (
|
|
234
|
+
f"{epoch+1:<6} | "
|
|
235
|
+
f"{train_loss:<10.4f} | "
|
|
236
|
+
f"{val_loss:<10.4f} | "
|
|
237
|
+
f"{ari:<6.3f} {nmi:<6.3f} {asw:<6.3f} | "
|
|
238
|
+
f"{cal:<6.3f} {dav:<6.3f} {cor:<6.3f} | "
|
|
239
|
+
f"{pat_status:<9}"
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
# This will appear just above the bar, under the header
|
|
243
|
+
pbar.write(log_msg)
|
|
244
|
+
|
|
245
|
+
# Compact info on the moving bar itself
|
|
246
|
+
postfix = {
|
|
247
|
+
"trn": f"{train_loss:.3f}",
|
|
248
|
+
"val": f"{val_loss:.3f}",
|
|
249
|
+
}
|
|
250
|
+
if early_stop:
|
|
251
|
+
postfix["best"] = f"{self.best_val_loss:.3f}"
|
|
252
|
+
pbar.set_postfix(postfix)
|
|
253
|
+
|
|
254
|
+
if should_stop and early_stop:
|
|
255
|
+
self.actual_epochs = epoch + 1
|
|
256
|
+
pbar.write(f"\n>>> Early stopping triggered at epoch {epoch + 1}")
|
|
257
|
+
break
|
|
258
|
+
|
|
259
|
+
# Update the bar each epoch
|
|
260
|
+
pbar.update(1)
|
|
261
|
+
else:
|
|
262
|
+
self.actual_epochs = epochs
|
|
244
263
|
|
|
245
264
|
# Record resource usage
|
|
246
265
|
self.train_time = time.time() - start_time
|
|
247
266
|
self.peak_memory_gb = torch.cuda.max_memory_allocated() / 1e9 if use_cuda else 0.0
|
|
248
267
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
print(f"{'='*70}")
|
|
252
|
-
print(f" Epochs: {self.actual_epochs}/{epochs}")
|
|
253
|
-
print(f" Time: {self.train_time:.2f}s ({self.train_time/self.actual_epochs:.2f}s/epoch)")
|
|
254
|
-
print(f" Peak GPU Memory: {self.peak_memory_gb:.3f} GB")
|
|
255
|
-
if early_stop:
|
|
256
|
-
print(f" Best Val Loss: {self.best_val_loss:.4f}")
|
|
257
|
-
print(f"{'='*70}\n")
|
|
268
|
+
# Final summary (after bar is closed, plain prints are fine)
|
|
269
|
+
self._print_training_summary(early_stop)
|
|
258
270
|
|
|
259
271
|
return self
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _print_training_summary(self, early_stop: bool):
|
|
275
|
+
"""Helper to print a clean final summary."""
|
|
276
|
+
print(f"\n{'='*60}")
|
|
277
|
+
print(f"{'Training Summary':^60}")
|
|
278
|
+
print(f"{'='*60}")
|
|
279
|
+
|
|
280
|
+
rows = [
|
|
281
|
+
("Total Epochs", f"{self.actual_epochs}"),
|
|
282
|
+
("Total Time", f"{self.train_time:.2f}s"),
|
|
283
|
+
("Time per Epoch", f"{self.train_time/self.actual_epochs:.3f}s"),
|
|
284
|
+
("Peak GPU Memory", f"{self.peak_memory_gb:.3f} GB"),
|
|
285
|
+
]
|
|
286
|
+
if early_stop:
|
|
287
|
+
rows.append(("Best Val Loss", f"{self.best_val_loss:.4f}"))
|
|
288
|
+
|
|
289
|
+
for label, value in rows:
|
|
290
|
+
print(f" {label:<20} : {value}")
|
|
291
|
+
print(f"{'='*60}\n")
|
|
292
|
+
|
|
260
293
|
# ========================================================================
|
|
261
294
|
# Representation Extraction
|
|
262
295
|
# ========================================================================
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
|
|
2
|
+
"""
|
|
3
|
+
Dataset utilities for iAODE examples.
|
|
4
|
+
|
|
5
|
+
Provides automatic download and caching of example datasets,
|
|
6
|
+
including scATAC-seq data and reference annotations.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import shutil
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Tuple, List
|
|
13
|
+
import requests
|
|
14
|
+
from tqdm import tqdm
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def get_data_dir() -> Path:
|
|
18
|
+
"""Get the iAODE data cache directory (fallback only)."""
|
|
19
|
+
data_dir = Path.home() / ".iaode" / "data"
|
|
20
|
+
data_dir.mkdir(parents=True, exist_ok=True)
|
|
21
|
+
(data_dir / "atacseq").mkdir(exist_ok=True)
|
|
22
|
+
(data_dir / "annotations").mkdir(exist_ok=True)
|
|
23
|
+
return data_dir
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _download_file(url: str, output_path: Path, desc: str = "file") -> bool:
|
|
27
|
+
"""
|
|
28
|
+
Download a file with progress indication.
|
|
29
|
+
|
|
30
|
+
Returns
|
|
31
|
+
-------
|
|
32
|
+
bool
|
|
33
|
+
True if download succeeded, False otherwise
|
|
34
|
+
"""
|
|
35
|
+
print(f"📥 Attempting download: {desc}")
|
|
36
|
+
print(f" URL: {url}")
|
|
37
|
+
|
|
38
|
+
try:
|
|
39
|
+
headers = {
|
|
40
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
response = requests.get(url, headers=headers, stream=True, timeout=30)
|
|
44
|
+
response.raise_for_status()
|
|
45
|
+
|
|
46
|
+
total_size = int(response.headers.get('content-length', 0))
|
|
47
|
+
block_size = 8192
|
|
48
|
+
|
|
49
|
+
with open(output_path, 'wb') as f, tqdm(
|
|
50
|
+
desc=desc,
|
|
51
|
+
total=total_size,
|
|
52
|
+
unit='iB',
|
|
53
|
+
unit_scale=True,
|
|
54
|
+
unit_divisor=1024,
|
|
55
|
+
) as pbar:
|
|
56
|
+
for chunk in response.iter_content(chunk_size=block_size):
|
|
57
|
+
if chunk:
|
|
58
|
+
size = f.write(chunk)
|
|
59
|
+
pbar.update(size)
|
|
60
|
+
|
|
61
|
+
print(" ✓ Downloaded successfully")
|
|
62
|
+
return True
|
|
63
|
+
|
|
64
|
+
except Exception as e:
|
|
65
|
+
print(f" ✗ Download failed: {e}")
|
|
66
|
+
if output_path.exists():
|
|
67
|
+
output_path.unlink()
|
|
68
|
+
return False
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _try_multiple_urls(urls: List[str], output_path: Path, desc: str) -> bool:
|
|
72
|
+
"""Try downloading from multiple URLs until one succeeds."""
|
|
73
|
+
for i, url in enumerate(urls, 1):
|
|
74
|
+
print(f"\n[Attempt {i}/{len(urls)}]")
|
|
75
|
+
if _download_file(url, output_path, desc):
|
|
76
|
+
return True
|
|
77
|
+
return False
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _decompress_gz(gz_path: Path, output_path: Path) -> None:
|
|
81
|
+
"""Decompress a gzipped file."""
|
|
82
|
+
import gzip
|
|
83
|
+
print(f"📦 Decompressing {gz_path.name}...")
|
|
84
|
+
with gzip.open(gz_path, 'rb') as f_in:
|
|
85
|
+
with open(output_path, 'wb') as f_out:
|
|
86
|
+
shutil.copyfileobj(f_in, f_out)
|
|
87
|
+
print(f" ✓ Decompressed to {output_path.name}")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def mouse_brain_5k_atacseq(force_download: bool = False) -> Tuple[Path, Path]:
|
|
91
|
+
"""
|
|
92
|
+
Get 10X Mouse Brain 5k scATAC-seq dataset and mouse annotation.
|
|
93
|
+
|
|
94
|
+
Priority order:
|
|
95
|
+
1. Local data/ folder (current or parent directory)
|
|
96
|
+
2. Cache (~/.iaode/data/)
|
|
97
|
+
3. Download to cache
|
|
98
|
+
|
|
99
|
+
Parameters
|
|
100
|
+
----------
|
|
101
|
+
force_download : bool
|
|
102
|
+
Re-download to cache even if files exist
|
|
103
|
+
|
|
104
|
+
Returns
|
|
105
|
+
-------
|
|
106
|
+
h5_path, gtf_path : Tuple[Path, Path]
|
|
107
|
+
Paths to H5 and GTF files
|
|
108
|
+
"""
|
|
109
|
+
# Step 1: Check local data folders first
|
|
110
|
+
local_search_dirs = [
|
|
111
|
+
Path.cwd() / "data",
|
|
112
|
+
Path.cwd().parent / "data",
|
|
113
|
+
Path.cwd() / "examples" / "data"
|
|
114
|
+
]
|
|
115
|
+
|
|
116
|
+
for local_dir in local_search_dirs:
|
|
117
|
+
local_h5 = local_dir / "mouse_brain_5k_v1.1.h5"
|
|
118
|
+
local_gtf = local_dir / "gencode.vM25.annotation.gtf"
|
|
119
|
+
|
|
120
|
+
if local_h5.exists() and local_gtf.exists():
|
|
121
|
+
print("=" * 70)
|
|
122
|
+
print("✅ Using local files (no download needed)")
|
|
123
|
+
print("=" * 70)
|
|
124
|
+
print(f"Location: {local_dir}")
|
|
125
|
+
print(f"H5: {local_h5.name}")
|
|
126
|
+
print(f"GTF: {local_gtf.name}")
|
|
127
|
+
print("=" * 70 + "\n")
|
|
128
|
+
return local_h5, local_gtf
|
|
129
|
+
|
|
130
|
+
# Step 2: Check cache if not forced to re-download
|
|
131
|
+
data_dir = get_data_dir()
|
|
132
|
+
h5_file = data_dir / "atacseq" / "mouse_brain_5k_v1.1.h5"
|
|
133
|
+
gtf_file = data_dir / "annotations" / "gencode.vM25.annotation.gtf"
|
|
134
|
+
gtf_gz = data_dir / "annotations" / "gencode.vM25.annotation.gtf.gz"
|
|
135
|
+
|
|
136
|
+
if not force_download and h5_file.exists() and gtf_file.exists():
|
|
137
|
+
print(f"✓ Using cached files: {data_dir}")
|
|
138
|
+
return h5_file, gtf_file
|
|
139
|
+
|
|
140
|
+
# Step 3: Download to cache
|
|
141
|
+
print("Local files not found. Downloading to cache...")
|
|
142
|
+
|
|
143
|
+
# Download H5 if needed
|
|
144
|
+
if not h5_file.exists() or force_download:
|
|
145
|
+
h5_urls = [
|
|
146
|
+
"https://cf.10xgenomics.com/samples/cell-atac/2.0.0/atac_mouse_brain_5k_v1.1/atac_mouse_brain_5k_v1.1_filtered_peak_bc_matrix.h5",
|
|
147
|
+
"https://cf.10xgenomics.com/samples/cell-atac/1.2.0/atac_mouse_brain_5k_v1.1/atac_mouse_brain_5k_v1.1_filtered_peak_bc_matrix.h5",
|
|
148
|
+
]
|
|
149
|
+
|
|
150
|
+
if not _try_multiple_urls(h5_urls, h5_file, "Mouse Brain 5k H5"):
|
|
151
|
+
print("\n" + "=" * 70)
|
|
152
|
+
print("⚠️ AUTOMATIC DOWNLOAD FAILED")
|
|
153
|
+
print("=" * 70)
|
|
154
|
+
print("Please download manually:")
|
|
155
|
+
print("\n1. Visit: https://www.10xgenomics.com/datasets")
|
|
156
|
+
print(" Search: 'Fresh Cortex from Adult Mouse Brain'")
|
|
157
|
+
print(f"\n2. Download 'Filtered peak-barcode matrix HDF5'")
|
|
158
|
+
print(f"\n3. Save to one of these locations:")
|
|
159
|
+
print(f" • {Path.cwd() / 'data' / 'mouse_brain_5k_v1.1.h5'}")
|
|
160
|
+
print(f" • {h5_file}")
|
|
161
|
+
print("=" * 70)
|
|
162
|
+
raise RuntimeError("Manual download required")
|
|
163
|
+
|
|
164
|
+
# Download GTF if needed
|
|
165
|
+
if not gtf_file.exists() or force_download:
|
|
166
|
+
if not gtf_gz.exists() or force_download:
|
|
167
|
+
gtf_urls = [
|
|
168
|
+
"https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M25/gencode.vM25.annotation.gtf.gz",
|
|
169
|
+
"ftp://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M25/gencode.vM25.annotation.gtf.gz"
|
|
170
|
+
]
|
|
171
|
+
|
|
172
|
+
if not _try_multiple_urls(gtf_urls, gtf_gz, "GENCODE vM25 GTF"):
|
|
173
|
+
print("\n⚠️ GTF download failed. Please download manually:")
|
|
174
|
+
print(f" URL: {gtf_urls[0]}")
|
|
175
|
+
print(f" Save to: {Path.cwd() / 'data' / 'gencode.vM25.annotation.gtf.gz'}")
|
|
176
|
+
raise RuntimeError("Manual GTF download required")
|
|
177
|
+
|
|
178
|
+
_decompress_gz(gtf_gz, gtf_file)
|
|
179
|
+
if gtf_gz.exists():
|
|
180
|
+
gtf_gz.unlink()
|
|
181
|
+
|
|
182
|
+
print("\n" + "=" * 70)
|
|
183
|
+
print("✅ Dataset ready (cached)")
|
|
184
|
+
print("=" * 70)
|
|
185
|
+
print(f"Cache: {data_dir}")
|
|
186
|
+
print(f"H5: {h5_file.name}")
|
|
187
|
+
print(f"GTF: {gtf_file.name}")
|
|
188
|
+
print("=" * 70 + "\n")
|
|
189
|
+
|
|
190
|
+
return h5_file, gtf_file
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def human_pbmc_5k_atacseq(force_download: bool = False) -> Tuple[Path, Path]:
|
|
194
|
+
"""
|
|
195
|
+
Get 10X Human PBMC 5k scATAC-seq dataset.
|
|
196
|
+
|
|
197
|
+
Priority: local data/ folder → cache → download
|
|
198
|
+
"""
|
|
199
|
+
# Check local first
|
|
200
|
+
local_search_dirs = [
|
|
201
|
+
Path.cwd() / "data",
|
|
202
|
+
Path.cwd().parent / "data",
|
|
203
|
+
Path.cwd() / "examples" / "data"
|
|
204
|
+
]
|
|
205
|
+
|
|
206
|
+
for local_dir in local_search_dirs:
|
|
207
|
+
local_h5 = local_dir / "human_pbmc_5k_nextgem.h5"
|
|
208
|
+
local_gtf = local_dir / "gencode.v49.annotation.gtf"
|
|
209
|
+
|
|
210
|
+
if local_h5.exists() and local_gtf.exists():
|
|
211
|
+
print(f"✓ Using local files: {local_dir}")
|
|
212
|
+
return local_h5, local_gtf
|
|
213
|
+
|
|
214
|
+
# Check cache
|
|
215
|
+
data_dir = get_data_dir()
|
|
216
|
+
h5_file = data_dir / "atacseq" / "human_pbmc_5k_nextgem.h5"
|
|
217
|
+
gtf_file = data_dir / "annotations" / "gencode.v49.annotation.gtf"
|
|
218
|
+
gtf_gz = data_dir / "annotations" / "gencode.v49.annotation.gtf.gz"
|
|
219
|
+
|
|
220
|
+
if not force_download and h5_file.exists() and gtf_file.exists():
|
|
221
|
+
print(f"✓ Using cached files: {data_dir}")
|
|
222
|
+
return h5_file, gtf_file
|
|
223
|
+
|
|
224
|
+
# Download to cache
|
|
225
|
+
print("Local files not found. Downloading to cache...")
|
|
226
|
+
|
|
227
|
+
if not h5_file.exists() or force_download:
|
|
228
|
+
h5_urls = [
|
|
229
|
+
"https://cf.10xgenomics.com/samples/cell-atac/2.0.0/atac_pbmc_5k_nextgem/atac_pbmc_5k_nextgem_filtered_peak_bc_matrix.h5",
|
|
230
|
+
"https://cf.10xgenomics.com/samples/cell-atac/1.1.0/atac_pbmc_5k_nextgem/atac_pbmc_5k_nextgem_filtered_peak_bc_matrix.h5",
|
|
231
|
+
]
|
|
232
|
+
|
|
233
|
+
if not _try_multiple_urls(h5_urls, h5_file, "PBMC 5k H5"):
|
|
234
|
+
print("\n⚠️ Download failed. Save manually to:")
|
|
235
|
+
print(f" {Path.cwd() / 'data' / 'human_pbmc_5k_nextgem.h5'}")
|
|
236
|
+
raise RuntimeError("Manual download required")
|
|
237
|
+
|
|
238
|
+
if not gtf_file.exists() or force_download:
|
|
239
|
+
if not gtf_gz.exists() or force_download:
|
|
240
|
+
gtf_urls = [
|
|
241
|
+
"https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_49/gencode.v49.annotation.gtf.gz",
|
|
242
|
+
]
|
|
243
|
+
_try_multiple_urls(gtf_urls, gtf_gz, "GENCODE v49 GTF")
|
|
244
|
+
|
|
245
|
+
_decompress_gz(gtf_gz, gtf_file)
|
|
246
|
+
if gtf_gz.exists():
|
|
247
|
+
gtf_gz.unlink()
|
|
248
|
+
|
|
249
|
+
print(f"\n✅ Dataset ready: {h5_file.name}, {gtf_file.name}\n")
|
|
250
|
+
return h5_file, gtf_file
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def clear_cache() -> None:
|
|
254
|
+
"""Clear the iAODE data cache (~/.iaode/data/)."""
|
|
255
|
+
data_dir = get_data_dir()
|
|
256
|
+
if data_dir.exists():
|
|
257
|
+
shutil.rmtree(data_dir)
|
|
258
|
+
print(f"✓ Cleared cache: {data_dir}")
|
|
259
|
+
else:
|
|
260
|
+
print("✓ Cache already empty")
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def list_cached_files() -> None:
|
|
264
|
+
"""List all cached dataset files."""
|
|
265
|
+
data_dir = get_data_dir()
|
|
266
|
+
print("=" * 70)
|
|
267
|
+
print("iAODE Cached Datasets")
|
|
268
|
+
print("=" * 70)
|
|
269
|
+
print(f"Cache location: {data_dir}")
|
|
270
|
+
print("\nNote: Local data/ folders are checked first and not listed here.\n")
|
|
271
|
+
|
|
272
|
+
if not data_dir.exists():
|
|
273
|
+
print("Cache is empty.\n")
|
|
274
|
+
print("=" * 70)
|
|
275
|
+
return
|
|
276
|
+
|
|
277
|
+
total_size = 0.0
|
|
278
|
+
file_count = 0
|
|
279
|
+
|
|
280
|
+
for root, dirs, files in os.walk(data_dir):
|
|
281
|
+
for file in files:
|
|
282
|
+
filepath = Path(root) / file
|
|
283
|
+
size_mb = filepath.stat().st_size / 1024 / 1024
|
|
284
|
+
total_size += size_mb
|
|
285
|
+
file_count += 1
|
|
286
|
+
print(f" {filepath.relative_to(data_dir)}")
|
|
287
|
+
print(f" {size_mb:.1f} MB")
|
|
288
|
+
|
|
289
|
+
if file_count == 0:
|
|
290
|
+
print("Cache is empty.\n")
|
|
291
|
+
else:
|
|
292
|
+
print(f"\nTotal: {file_count} files, {total_size:.1f} MB")
|
|
293
|
+
|
|
294
|
+
print("=" * 70)
|
|
@@ -1,5 +1,9 @@
|
|
|
1
|
+
"""
|
|
2
|
+
scATAC-seq Preprocessing Utilities and FastAPI Endpoints
|
|
3
|
+
"""
|
|
4
|
+
|
|
1
5
|
import numpy as np
|
|
2
|
-
from scipy.sparse import issparse, csr_matrix
|
|
6
|
+
from scipy.sparse import issparse, csr_matrix
|
|
3
7
|
from typing import Literal, Tuple, Optional
|
|
4
8
|
import warnings
|
|
5
9
|
|
|
@@ -44,11 +48,14 @@ def tfidf_normalization(adata,
|
|
|
44
48
|
|
|
45
49
|
print(f"Applying TF-IDF normalization (scale={scale_factor:.0e})...")
|
|
46
50
|
|
|
47
|
-
# Get counts matrix
|
|
51
|
+
# Get counts matrix - ensure CSR format
|
|
48
52
|
if issparse(adata.X):
|
|
49
|
-
|
|
53
|
+
if adata.X.format != 'csr':
|
|
54
|
+
adata.X = adata.X.tocsr()
|
|
55
|
+
X = adata.X
|
|
50
56
|
else:
|
|
51
|
-
X = csr_matrix(adata.X
|
|
57
|
+
X = csr_matrix(adata.X)
|
|
58
|
+
adata.X = X
|
|
52
59
|
|
|
53
60
|
# ========================================================================
|
|
54
61
|
# Term Frequency (TF)
|
|
@@ -58,12 +65,14 @@ def tfidf_normalization(adata,
|
|
|
58
65
|
cell_sums = np.array(X.sum(axis=1)).flatten()
|
|
59
66
|
cell_sums[cell_sums == 0] = 1 # Avoid division by zero
|
|
60
67
|
|
|
61
|
-
#
|
|
62
|
-
|
|
68
|
+
# In-place normalization: peak_count / total_counts_per_cell
|
|
69
|
+
# Map each data point to its row's cell_sum
|
|
70
|
+
row_indices = np.repeat(np.arange(X.shape[0]), np.diff(X.indptr))
|
|
71
|
+
X.data /= cell_sums[row_indices]
|
|
63
72
|
|
|
64
73
|
# Optional: log-transform TF (log(1 + TF))
|
|
65
74
|
if log_tf:
|
|
66
|
-
|
|
75
|
+
np.log1p(X.data, out=X.data)
|
|
67
76
|
print(" Using log-transformed TF: log(1 + TF)")
|
|
68
77
|
|
|
69
78
|
# ========================================================================
|
|
@@ -73,7 +82,7 @@ def tfidf_normalization(adata,
|
|
|
73
82
|
n_cells = adata.n_obs
|
|
74
83
|
|
|
75
84
|
# Count cells where each peak is accessible (count > 0)
|
|
76
|
-
n_cells_per_peak = np.asarray((X > 0).sum(axis=0)).ravel()
|
|
85
|
+
n_cells_per_peak = np.asarray((X > 0).sum(axis=0)).ravel()
|
|
77
86
|
n_cells_per_peak[n_cells_per_peak == 0] = 1 # Avoid division by zero
|
|
78
87
|
|
|
79
88
|
if log_idf:
|
|
@@ -87,10 +96,11 @@ def tfidf_normalization(adata,
|
|
|
87
96
|
# TF-IDF = TF * IDF * scale_factor
|
|
88
97
|
# ========================================================================
|
|
89
98
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
99
|
+
# In-place multiplication: use column indices to map IDF values
|
|
100
|
+
X.data *= idf[X.indices]
|
|
101
|
+
|
|
102
|
+
# In-place scaling
|
|
103
|
+
X.data *= scale_factor
|
|
94
104
|
|
|
95
105
|
# Store normalization parameters
|
|
96
106
|
adata.uns['tfidf_params'] = {
|
|
@@ -99,7 +109,7 @@ def tfidf_normalization(adata,
|
|
|
99
109
|
'log_idf': log_idf
|
|
100
110
|
}
|
|
101
111
|
|
|
102
|
-
print(f" TF-IDF complete. Value range: [{
|
|
112
|
+
print(f" TF-IDF complete. Value range: [{X.data.min():.2e}, {X.data.max():.2e}]")
|
|
103
113
|
|
|
104
114
|
return adata if not inplace else None
|
|
105
115
|
|
|
@@ -305,6 +315,76 @@ def select_highly_variable_peaks(
|
|
|
305
315
|
return adata if not inplace else None
|
|
306
316
|
|
|
307
317
|
|
|
318
|
+
# ============================================================================
|
|
319
|
+
# UTILITY FUNCTION: Subsampling
|
|
320
|
+
# ============================================================================
|
|
321
|
+
|
|
322
|
+
def subsample_cells_and_peaks(
|
|
323
|
+
adata,
|
|
324
|
+
n_cells: Optional[int] = None,
|
|
325
|
+
frac_cells: Optional[float] = None,
|
|
326
|
+
use_hvp: bool = True,
|
|
327
|
+
hvp_column: str = 'highly_variable',
|
|
328
|
+
seed: int = 42,
|
|
329
|
+
inplace: bool = False
|
|
330
|
+
):
|
|
331
|
+
"""
|
|
332
|
+
Subsample cells and optionally filter to highly variable peaks
|
|
333
|
+
|
|
334
|
+
Args:
|
|
335
|
+
adata: AnnData object
|
|
336
|
+
n_cells: Number of cells to sample (if None, use frac_cells)
|
|
337
|
+
frac_cells: Fraction of cells to sample (if n_cells is None)
|
|
338
|
+
use_hvp: Whether to filter to highly variable peaks
|
|
339
|
+
hvp_column: Column name in adata.var for highly variable peaks
|
|
340
|
+
seed: Random seed for reproducibility
|
|
341
|
+
inplace: Modify adata in place
|
|
342
|
+
|
|
343
|
+
Returns:
|
|
344
|
+
Subsampled AnnData (or None if inplace=True)
|
|
345
|
+
"""
|
|
346
|
+
|
|
347
|
+
if not inplace:
|
|
348
|
+
adata = adata.copy()
|
|
349
|
+
|
|
350
|
+
# Determine number of cells to sample
|
|
351
|
+
if n_cells is None and frac_cells is None:
|
|
352
|
+
raise ValueError("Must specify either n_cells or frac_cells")
|
|
353
|
+
|
|
354
|
+
if n_cells is None:
|
|
355
|
+
n_cells = int(adata.n_obs * frac_cells)
|
|
356
|
+
|
|
357
|
+
n_cells = min(n_cells, adata.n_obs)
|
|
358
|
+
|
|
359
|
+
print(f"\nSubsampling to {n_cells:,} cells...")
|
|
360
|
+
|
|
361
|
+
# Random cell selection
|
|
362
|
+
np.random.seed(seed)
|
|
363
|
+
cell_idxs = np.random.choice(adata.n_obs, n_cells, replace=False)
|
|
364
|
+
|
|
365
|
+
# Peak filtering
|
|
366
|
+
if use_hvp:
|
|
367
|
+
if hvp_column not in adata.var.columns:
|
|
368
|
+
raise ValueError(f"Column '{hvp_column}' not found in adata.var")
|
|
369
|
+
|
|
370
|
+
peak_mask = adata.var[hvp_column].values
|
|
371
|
+
n_peaks = peak_mask.sum()
|
|
372
|
+
print(f"Filtering to {n_peaks:,} highly variable peaks...")
|
|
373
|
+
|
|
374
|
+
result = adata[cell_idxs, peak_mask]
|
|
375
|
+
else:
|
|
376
|
+
result = adata[cell_idxs, :]
|
|
377
|
+
|
|
378
|
+
print(f"Final shape: {result.n_obs} cells × {result.n_vars} peaks")
|
|
379
|
+
|
|
380
|
+
if inplace:
|
|
381
|
+
# Update adata in place
|
|
382
|
+
adata._inplace_subset_obs(cell_idxs)
|
|
383
|
+
if use_hvp:
|
|
384
|
+
adata._inplace_subset_var(peak_mask)
|
|
385
|
+
return None
|
|
386
|
+
else:
|
|
387
|
+
return result.copy()
|
|
308
388
|
|
|
309
389
|
def get_dataset_category(n_cells: int) -> Tuple[str, Optional[int], Optional[int]]:
|
|
310
390
|
"""
|
|
@@ -323,5 +403,4 @@ def get_dataset_category(n_cells: int) -> Tuple[str, Optional[int], Optional[int
|
|
|
323
403
|
elif n_cells < 20000:
|
|
324
404
|
return 'medium', 10000, 20000
|
|
325
405
|
else:
|
|
326
|
-
return 'large', 20000, 20000
|
|
327
|
-
|
|
406
|
+
return 'large', 20000, 20000
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: iaode
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.9
|
|
4
4
|
Summary: Interpretable Autoencoder with Ordinary Differential Equations for single-cell omics analysis
|
|
5
5
|
Author-email: Zeyu Fu <fuzeyu99@126.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -98,34 +98,40 @@ Loss = recon + i_recon + ODE_consistency + β·KL + dip·DIP + tc·TC + info·MM
|
|
|
98
98
|
|
|
99
99
|
## Installation
|
|
100
100
|
|
|
101
|
-
###
|
|
102
|
-
|
|
101
|
+
### Method 1: Python Library (Standard)
|
|
102
|
+
For users using iAODE in scripts or notebooks:
|
|
103
103
|
```bash
|
|
104
104
|
pip install iaode
|
|
105
105
|
```
|
|
106
106
|
|
|
107
|
-
###
|
|
107
|
+
### Method 2: Web Interface (Full)
|
|
108
|
+
For users who want the **Interactive Training UI**, you must clone the repository (since the UI requires the server files):
|
|
108
109
|
|
|
109
110
|
```bash
|
|
110
111
|
git clone https://github.com/PeterPonyu/iAODE.git
|
|
111
112
|
cd iAODE
|
|
112
113
|
pip install -e .
|
|
114
|
+
pip install fastapi uvicorn # Required for the UI server
|
|
113
115
|
```
|
|
114
116
|
|
|
115
|
-
### Requirements
|
|
116
|
-
|
|
117
|
-
- Python ≥ 3.9
|
|
118
|
-
- PyTorch ≥ 1.10.0
|
|
119
|
-
- AnnData ≥ 0.8.0
|
|
120
|
-
- Scanpy ≥ 1.8.0
|
|
121
|
-
- scvi-tools ≥ 0.16.0
|
|
122
|
-
|
|
123
|
-
See `requirements.txt` for the complete list of dependencies.
|
|
124
|
-
|
|
125
117
|
---
|
|
126
|
-
|
|
127
118
|
## Quick Start
|
|
128
119
|
|
|
120
|
+
### 🖥️ Interactive Web Interface
|
|
121
|
+
**No Node.js required.** The interface is pre-built and served via Python.
|
|
122
|
+
|
|
123
|
+
1. **Start the Server:**
|
|
124
|
+
```bash
|
|
125
|
+
python api/run_server.py
|
|
126
|
+
```
|
|
127
|
+
2. **Access the UI:**
|
|
128
|
+
Open **[http://localhost:8000/](http://localhost:8000/)** in your browser.
|
|
129
|
+
|
|
130
|
+
*Features:*
|
|
131
|
+
* Upload `.h5ad` or `.h5` files.
|
|
132
|
+
* Configure Encoders, Loss functions, and ODE parameters visually.
|
|
133
|
+
* Download trained embeddings directly.
|
|
134
|
+
|
|
129
135
|
### Basic scATAC-seq Workflow
|
|
130
136
|
|
|
131
137
|
```python
|
iaode-0.2.7/iaode/datasets.py
DELETED
|
@@ -1,329 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Dataset utilities for iAODE examples.
|
|
3
|
-
|
|
4
|
-
Provides automatic download and caching of example datasets,
|
|
5
|
-
including scATAC-seq data and reference annotations.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
import os
|
|
9
|
-
import urllib.request
|
|
10
|
-
import gzip
|
|
11
|
-
import shutil
|
|
12
|
-
from pathlib import Path
|
|
13
|
-
from typing import Tuple
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def get_data_dir() -> Path:
|
|
17
|
-
"""
|
|
18
|
-
Get the iAODE data cache directory.
|
|
19
|
-
|
|
20
|
-
Creates directory structure:
|
|
21
|
-
~/.iaode/data/
|
|
22
|
-
├── atacseq/
|
|
23
|
-
└── annotations/
|
|
24
|
-
|
|
25
|
-
Returns
|
|
26
|
-
-------
|
|
27
|
-
Path
|
|
28
|
-
Path to data cache directory
|
|
29
|
-
"""
|
|
30
|
-
data_dir = Path.home() / ".iaode" / "data"
|
|
31
|
-
data_dir.mkdir(parents=True, exist_ok=True)
|
|
32
|
-
(data_dir / "atacseq").mkdir(exist_ok=True)
|
|
33
|
-
(data_dir / "annotations").mkdir(exist_ok=True)
|
|
34
|
-
return data_dir
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def _download_file(url: str, output_path: Path, desc: str = "file") -> None:
|
|
38
|
-
"""
|
|
39
|
-
Download a file with progress indication.
|
|
40
|
-
|
|
41
|
-
Parameters
|
|
42
|
-
----------
|
|
43
|
-
url : str
|
|
44
|
-
URL to download from
|
|
45
|
-
output_path : Path
|
|
46
|
-
Local path to save file
|
|
47
|
-
desc : str
|
|
48
|
-
Description for progress messages
|
|
49
|
-
"""
|
|
50
|
-
print(f"📥 Downloading {desc}...")
|
|
51
|
-
print(f" URL: {url}")
|
|
52
|
-
print(f" → {output_path}")
|
|
53
|
-
|
|
54
|
-
try:
|
|
55
|
-
# Create a custom progress hook
|
|
56
|
-
def reporthook(blocknum, blocksize, totalsize):
|
|
57
|
-
if totalsize > 0:
|
|
58
|
-
percent = min(blocknum * blocksize / totalsize * 100, 100)
|
|
59
|
-
mb_downloaded = blocknum * blocksize / 1024 / 1024
|
|
60
|
-
mb_total = totalsize / 1024 / 1024
|
|
61
|
-
print(f"\r Progress: {percent:.1f}% ({mb_downloaded:.1f}/{mb_total:.1f} MB)", end='')
|
|
62
|
-
|
|
63
|
-
urllib.request.urlretrieve(url, output_path, reporthook=reporthook)
|
|
64
|
-
print() # New line after progress
|
|
65
|
-
print(" ✓ Downloaded successfully")
|
|
66
|
-
except Exception as e:
|
|
67
|
-
print(f"\n ✗ Download failed: {e}")
|
|
68
|
-
raise
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
def _decompress_gz(gz_path: Path, output_path: Path) -> None:
|
|
72
|
-
"""
|
|
73
|
-
Decompress a gzipped file.
|
|
74
|
-
|
|
75
|
-
Parameters
|
|
76
|
-
----------
|
|
77
|
-
gz_path : Path
|
|
78
|
-
Path to .gz file
|
|
79
|
-
output_path : Path
|
|
80
|
-
Path for decompressed output
|
|
81
|
-
"""
|
|
82
|
-
print(f"📦 Decompressing {gz_path.name}...")
|
|
83
|
-
with gzip.open(gz_path, 'rb') as f_in:
|
|
84
|
-
with open(output_path, 'wb') as f_out:
|
|
85
|
-
shutil.copyfileobj(f_in, f_out)
|
|
86
|
-
print(f" ✓ Decompressed to {output_path.name}")
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def mouse_brain_5k_atacseq(force_download: bool = False, use_local: bool = True) -> Tuple[Path, Path]:
|
|
90
|
-
"""
|
|
91
|
-
Download 10X Mouse Brain 5k scATAC-seq dataset and mouse annotation.
|
|
92
|
-
|
|
93
|
-
This dataset contains ~5,000 cells from mouse brain tissue, profiled
|
|
94
|
-
with 10X Chromium scATAC-seq. Automatically downloads both the peak
|
|
95
|
-
matrix (H5 format) and the mouse GENCODE annotation (GTF).
|
|
96
|
-
|
|
97
|
-
Parameters
|
|
98
|
-
----------
|
|
99
|
-
force_download : bool, optional
|
|
100
|
-
If True, re-download even if files exist (default: False)
|
|
101
|
-
use_local : bool, optional
|
|
102
|
-
If True, check examples/data/ for files before downloading (default: True)
|
|
103
|
-
|
|
104
|
-
Returns
|
|
105
|
-
-------
|
|
106
|
-
h5_path : Path
|
|
107
|
-
Path to filtered_peak_bc_matrix.h5 file
|
|
108
|
-
gtf_path : Path
|
|
109
|
-
Path to gencode.vM25.annotation.gtf file
|
|
110
|
-
|
|
111
|
-
Examples
|
|
112
|
-
--------
|
|
113
|
-
>>> import iaode
|
|
114
|
-
>>> h5_file, gtf_file = iaode.datasets.mouse_brain_5k_atacseq()
|
|
115
|
-
>>> adata = iaode.annotation_pipeline(h5_file, gtf_file)
|
|
116
|
-
|
|
117
|
-
Notes
|
|
118
|
-
-----
|
|
119
|
-
If automatic download fails (403 error), the function will look for files in:
|
|
120
|
-
- Current directory: examples/data/
|
|
121
|
-
- User cache: ~/.iaode/data/
|
|
122
|
-
|
|
123
|
-
Manual download instructions will be provided if files are not found.
|
|
124
|
-
"""
|
|
125
|
-
data_dir = get_data_dir()
|
|
126
|
-
atacseq_dir = data_dir / "atacseq"
|
|
127
|
-
anno_dir = data_dir / "annotations"
|
|
128
|
-
|
|
129
|
-
# Define file paths
|
|
130
|
-
h5_file = atacseq_dir / "mouse_brain_5k_v1.1.h5"
|
|
131
|
-
gtf_file = anno_dir / "gencode.vM25.annotation.gtf"
|
|
132
|
-
gtf_gz = anno_dir / "gencode.vM25.annotation.gtf.gz"
|
|
133
|
-
|
|
134
|
-
# Check for local files first (e.g., in examples/data/)
|
|
135
|
-
if use_local:
|
|
136
|
-
local_examples_dir = Path.cwd() / "data"
|
|
137
|
-
if not local_examples_dir.exists():
|
|
138
|
-
local_examples_dir = Path.cwd().parent / "data" # Try parent dir
|
|
139
|
-
|
|
140
|
-
local_h5 = local_examples_dir / "mouse_brain_5k_v1.1.h5"
|
|
141
|
-
local_gtf = local_examples_dir / "gencode.vM25.annotation.gtf"
|
|
142
|
-
|
|
143
|
-
if local_h5.exists() and local_gtf.exists():
|
|
144
|
-
print(f"✓ Using local files from: {local_examples_dir}")
|
|
145
|
-
# Copy to cache for consistency
|
|
146
|
-
if not h5_file.exists():
|
|
147
|
-
print(" Copying H5 to cache...")
|
|
148
|
-
shutil.copy(local_h5, h5_file)
|
|
149
|
-
if not gtf_file.exists():
|
|
150
|
-
print(" Copying GTF to cache...")
|
|
151
|
-
shutil.copy(local_gtf, gtf_file)
|
|
152
|
-
print()
|
|
153
|
-
print("=" * 70)
|
|
154
|
-
print("✅ Dataset ready!")
|
|
155
|
-
print("=" * 70)
|
|
156
|
-
print(f"H5 file: {h5_file}")
|
|
157
|
-
print(f"GTF file: {gtf_file}")
|
|
158
|
-
print(f"Cache location: {data_dir}")
|
|
159
|
-
print("=" * 70)
|
|
160
|
-
print()
|
|
161
|
-
return h5_file, gtf_file
|
|
162
|
-
|
|
163
|
-
# Download H5 file if needed
|
|
164
|
-
if not h5_file.exists() or force_download:
|
|
165
|
-
h5_url = "https://cf.10xgenomics.com/samples/cell-atac/2.0.0/atac_mouse_brain_5k_v1.1/atac_mouse_brain_5k_v1.1_filtered_peak_bc_matrix.h5"
|
|
166
|
-
try:
|
|
167
|
-
_download_file(h5_url, h5_file, "mouse brain 5k scATAC-seq H5 matrix")
|
|
168
|
-
except Exception as e:
|
|
169
|
-
print(f"\n⚠️ Automatic download failed: {e}")
|
|
170
|
-
print("\n" + "=" * 70)
|
|
171
|
-
print("MANUAL DOWNLOAD REQUIRED")
|
|
172
|
-
print("=" * 70)
|
|
173
|
-
print("Please download the file manually:")
|
|
174
|
-
print("\n1. Visit: https://www.10xgenomics.com/datasets/fresh-cortex-from-adult-mouse-brain-p-50-1-standard-2-0-0")
|
|
175
|
-
print(" Or try direct link:")
|
|
176
|
-
print(f" {h5_url}")
|
|
177
|
-
print(f"\n2. Save as: {h5_file}")
|
|
178
|
-
print(f"\n3. Or place in: {Path.cwd() / 'data' / 'mouse_brain_5k_v1.1.h5'}")
|
|
179
|
-
print("=" * 70)
|
|
180
|
-
raise
|
|
181
|
-
else:
|
|
182
|
-
print(f"✓ Using cached H5: {h5_file}")
|
|
183
|
-
|
|
184
|
-
# Download GTF if needed
|
|
185
|
-
if not gtf_file.exists() or force_download:
|
|
186
|
-
if not gtf_gz.exists() or force_download:
|
|
187
|
-
gtf_url = "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M25/gencode.vM25.annotation.gtf.gz"
|
|
188
|
-
_download_file(gtf_url, gtf_gz, "GENCODE vM25 mouse annotation (847 MB compressed)")
|
|
189
|
-
|
|
190
|
-
# Decompress
|
|
191
|
-
_decompress_gz(gtf_gz, gtf_file)
|
|
192
|
-
|
|
193
|
-
# Clean up compressed file to save space
|
|
194
|
-
if gtf_gz.exists():
|
|
195
|
-
gtf_gz.unlink()
|
|
196
|
-
print(" ✓ Removed compressed file to save space")
|
|
197
|
-
else:
|
|
198
|
-
print(f"✓ Using cached GTF: {gtf_file}")
|
|
199
|
-
|
|
200
|
-
print()
|
|
201
|
-
print("=" * 70)
|
|
202
|
-
print("✅ Dataset ready!")
|
|
203
|
-
print("=" * 70)
|
|
204
|
-
print(f"H5 file: {h5_file}")
|
|
205
|
-
print(f"GTF file: {gtf_file}")
|
|
206
|
-
print(f"Cache location: {data_dir}")
|
|
207
|
-
print("=" * 70)
|
|
208
|
-
print()
|
|
209
|
-
|
|
210
|
-
return h5_file, gtf_file
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
def human_pbmc_5k_atacseq(force_download: bool = False) -> Tuple[Path, Path]:
|
|
214
|
-
"""
|
|
215
|
-
Download 10X Human PBMC 5k scATAC-seq dataset and human annotation.
|
|
216
|
-
|
|
217
|
-
This dataset contains ~5,000 peripheral blood mononuclear cells (PBMCs),
|
|
218
|
-
profiled with 10X Chromium scATAC-seq. Automatically downloads both the
|
|
219
|
-
peak matrix (H5 format) and the human GENCODE annotation (GTF).
|
|
220
|
-
|
|
221
|
-
Parameters
|
|
222
|
-
----------
|
|
223
|
-
force_download : bool, optional
|
|
224
|
-
If True, re-download even if files exist (default: False)
|
|
225
|
-
|
|
226
|
-
Returns
|
|
227
|
-
-------
|
|
228
|
-
h5_path : Path
|
|
229
|
-
Path to filtered_peak_bc_matrix.h5 file
|
|
230
|
-
gtf_path : Path
|
|
231
|
-
Path to gencode.v49.annotation.gtf file
|
|
232
|
-
|
|
233
|
-
Examples
|
|
234
|
-
--------
|
|
235
|
-
>>> import iaode
|
|
236
|
-
>>> h5_file, gtf_file = iaode.datasets.human_pbmc_5k_atacseq()
|
|
237
|
-
>>> adata = iaode.annotation_pipeline(h5_file, gtf_file)
|
|
238
|
-
"""
|
|
239
|
-
data_dir = get_data_dir()
|
|
240
|
-
atacseq_dir = data_dir / "atacseq"
|
|
241
|
-
anno_dir = data_dir / "annotations"
|
|
242
|
-
|
|
243
|
-
# Define file paths
|
|
244
|
-
h5_file = atacseq_dir / "human_pbmc_5k_nextgem.h5"
|
|
245
|
-
gtf_file = anno_dir / "gencode.v49.annotation.gtf"
|
|
246
|
-
gtf_gz = anno_dir / "gencode.v49.annotation.gtf.gz"
|
|
247
|
-
|
|
248
|
-
# Download H5 file if needed
|
|
249
|
-
if not h5_file.exists() or force_download:
|
|
250
|
-
h5_url = "https://cf.10xgenomics.com/samples/cell-atac/2.0.0/atac_pbmc_5k_nextgem/atac_pbmc_5k_nextgem_filtered_peak_bc_matrix.h5"
|
|
251
|
-
_download_file(h5_url, h5_file, "human PBMC 5k scATAC-seq H5 matrix")
|
|
252
|
-
else:
|
|
253
|
-
print(f"✓ Using cached H5: {h5_file}")
|
|
254
|
-
|
|
255
|
-
# Download GTF if needed
|
|
256
|
-
if not gtf_file.exists() or force_download:
|
|
257
|
-
if not gtf_gz.exists() or force_download:
|
|
258
|
-
gtf_url = "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_49/gencode.v49.annotation.gtf.gz"
|
|
259
|
-
_download_file(gtf_url, gtf_gz, "GENCODE v49 human annotation (1.4 GB compressed)")
|
|
260
|
-
|
|
261
|
-
# Decompress
|
|
262
|
-
_decompress_gz(gtf_gz, gtf_file)
|
|
263
|
-
|
|
264
|
-
# Clean up compressed file
|
|
265
|
-
if gtf_gz.exists():
|
|
266
|
-
gtf_gz.unlink()
|
|
267
|
-
print(" ✓ Removed compressed file to save space")
|
|
268
|
-
else:
|
|
269
|
-
print(f"✓ Using cached GTF: {gtf_file}")
|
|
270
|
-
|
|
271
|
-
print()
|
|
272
|
-
print("=" * 70)
|
|
273
|
-
print("✅ Dataset ready!")
|
|
274
|
-
print("=" * 70)
|
|
275
|
-
print(f"H5 file: {h5_file}")
|
|
276
|
-
print(f"GTF file: {gtf_file}")
|
|
277
|
-
print(f"Cache location: {data_dir}")
|
|
278
|
-
print("=" * 70)
|
|
279
|
-
print()
|
|
280
|
-
|
|
281
|
-
return h5_file, gtf_file
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
def clear_cache() -> None:
|
|
285
|
-
"""
|
|
286
|
-
Clear the iAODE data cache directory.
|
|
287
|
-
|
|
288
|
-
Removes all downloaded datasets to free disk space.
|
|
289
|
-
Files will be re-downloaded on next use.
|
|
290
|
-
"""
|
|
291
|
-
data_dir = get_data_dir()
|
|
292
|
-
if data_dir.exists():
|
|
293
|
-
shutil.rmtree(data_dir)
|
|
294
|
-
print(f"✓ Cleared cache: {data_dir}")
|
|
295
|
-
print(" Files will be re-downloaded when needed.")
|
|
296
|
-
else:
|
|
297
|
-
print("✓ Cache already empty")
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
def list_cached_files() -> None:
|
|
301
|
-
"""
|
|
302
|
-
List all cached dataset files and their sizes.
|
|
303
|
-
"""
|
|
304
|
-
data_dir = get_data_dir()
|
|
305
|
-
print("=" * 70)
|
|
306
|
-
print("iAODE Cached Datasets")
|
|
307
|
-
print("=" * 70)
|
|
308
|
-
print(f"Location: {data_dir}")
|
|
309
|
-
print()
|
|
310
|
-
|
|
311
|
-
total_size: float = 0.0
|
|
312
|
-
file_count = 0
|
|
313
|
-
|
|
314
|
-
for root, dirs, files in os.walk(data_dir):
|
|
315
|
-
for file in files:
|
|
316
|
-
filepath = Path(root) / file
|
|
317
|
-
size_mb = filepath.stat().st_size / 1024 / 1024
|
|
318
|
-
total_size += size_mb
|
|
319
|
-
file_count += 1
|
|
320
|
-
rel_path = filepath.relative_to(data_dir)
|
|
321
|
-
print(f" {rel_path}")
|
|
322
|
-
print(f" Size: {size_mb:.1f} MB")
|
|
323
|
-
|
|
324
|
-
print()
|
|
325
|
-
print(f"Total: {file_count} files, {total_size:.1f} MB")
|
|
326
|
-
print("=" * 70)
|
|
327
|
-
|
|
328
|
-
if file_count > 0:
|
|
329
|
-
print("\nTo clear cache: iaode.datasets.clear_cache()")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|