iaode 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
iaode/datasets.py ADDED
@@ -0,0 +1,330 @@
1
+ """
2
+ Dataset utilities for iAODE examples.
3
+
4
+ Provides automatic download and caching of example datasets,
5
+ including scATAC-seq data and reference annotations.
6
+ """
7
+
8
+ import os
9
+ import urllib.request
10
+ import gzip
11
+ import shutil
12
+ from pathlib import Path
13
+ from typing import Optional, Tuple
14
+ import anndata as ad
15
+
16
+
17
+ def get_data_dir() -> Path:
18
+ """
19
+ Get the iAODE data cache directory.
20
+
21
+ Creates directory structure:
22
+ ~/.iaode/data/
23
+ ├── atacseq/
24
+ └── annotations/
25
+
26
+ Returns
27
+ -------
28
+ Path
29
+ Path to data cache directory
30
+ """
31
+ data_dir = Path.home() / ".iaode" / "data"
32
+ data_dir.mkdir(parents=True, exist_ok=True)
33
+ (data_dir / "atacseq").mkdir(exist_ok=True)
34
+ (data_dir / "annotations").mkdir(exist_ok=True)
35
+ return data_dir
36
+
37
+
38
+ def _download_file(url: str, output_path: Path, desc: str = "file") -> None:
39
+ """
40
+ Download a file with progress indication.
41
+
42
+ Parameters
43
+ ----------
44
+ url : str
45
+ URL to download from
46
+ output_path : Path
47
+ Local path to save file
48
+ desc : str
49
+ Description for progress messages
50
+ """
51
+ print(f"📥 Downloading {desc}...")
52
+ print(f" URL: {url}")
53
+ print(f" → {output_path}")
54
+
55
+ try:
56
+ # Create a custom progress hook
57
+ def reporthook(blocknum, blocksize, totalsize):
58
+ if totalsize > 0:
59
+ percent = min(blocknum * blocksize / totalsize * 100, 100)
60
+ mb_downloaded = blocknum * blocksize / 1024 / 1024
61
+ mb_total = totalsize / 1024 / 1024
62
+ print(f"\r Progress: {percent:.1f}% ({mb_downloaded:.1f}/{mb_total:.1f} MB)", end='')
63
+
64
+ urllib.request.urlretrieve(url, output_path, reporthook=reporthook)
65
+ print() # New line after progress
66
+ print(f" ✓ Downloaded successfully")
67
+ except Exception as e:
68
+ print(f"\n ✗ Download failed: {e}")
69
+ raise
70
+
71
+
72
+ def _decompress_gz(gz_path: Path, output_path: Path) -> None:
73
+ """
74
+ Decompress a gzipped file.
75
+
76
+ Parameters
77
+ ----------
78
+ gz_path : Path
79
+ Path to .gz file
80
+ output_path : Path
81
+ Path for decompressed output
82
+ """
83
+ print(f"📦 Decompressing {gz_path.name}...")
84
+ with gzip.open(gz_path, 'rb') as f_in:
85
+ with open(output_path, 'wb') as f_out:
86
+ shutil.copyfileobj(f_in, f_out)
87
+ print(f" ✓ Decompressed to {output_path.name}")
88
+
89
+
90
+ def mouse_brain_5k_atacseq(force_download: bool = False, use_local: bool = True) -> Tuple[Path, Path]:
91
+ """
92
+ Download 10X Mouse Brain 5k scATAC-seq dataset and mouse annotation.
93
+
94
+ This dataset contains ~5,000 cells from mouse brain tissue, profiled
95
+ with 10X Chromium scATAC-seq. Automatically downloads both the peak
96
+ matrix (H5 format) and the mouse GENCODE annotation (GTF).
97
+
98
+ Parameters
99
+ ----------
100
+ force_download : bool, optional
101
+ If True, re-download even if files exist (default: False)
102
+ use_local : bool, optional
103
+ If True, check examples/data/ for files before downloading (default: True)
104
+
105
+ Returns
106
+ -------
107
+ h5_path : Path
108
+ Path to filtered_peak_bc_matrix.h5 file
109
+ gtf_path : Path
110
+ Path to gencode.vM25.annotation.gtf file
111
+
112
+ Examples
113
+ --------
114
+ >>> import iaode
115
+ >>> h5_file, gtf_file = iaode.datasets.mouse_brain_5k_atacseq()
116
+ >>> adata = iaode.annotation_pipeline(h5_file, gtf_file)
117
+
118
+ Notes
119
+ -----
120
+ If automatic download fails (403 error), the function will look for files in:
121
+ - Current directory: examples/data/
122
+ - User cache: ~/.iaode/data/
123
+
124
+ Manual download instructions will be provided if files are not found.
125
+ """
126
+ data_dir = get_data_dir()
127
+ atacseq_dir = data_dir / "atacseq"
128
+ anno_dir = data_dir / "annotations"
129
+
130
+ # Define file paths
131
+ h5_file = atacseq_dir / "mouse_brain_5k_v1.1.h5"
132
+ gtf_file = anno_dir / "gencode.vM25.annotation.gtf"
133
+ gtf_gz = anno_dir / "gencode.vM25.annotation.gtf.gz"
134
+
135
+ # Check for local files first (e.g., in examples/data/)
136
+ if use_local:
137
+ local_examples_dir = Path.cwd() / "data"
138
+ if not local_examples_dir.exists():
139
+ local_examples_dir = Path.cwd().parent / "data" # Try parent dir
140
+
141
+ local_h5 = local_examples_dir / "mouse_brain_5k_v1.1.h5"
142
+ local_gtf = local_examples_dir / "gencode.vM25.annotation.gtf"
143
+
144
+ if local_h5.exists() and local_gtf.exists():
145
+ print(f"✓ Using local files from: {local_examples_dir}")
146
+ # Copy to cache for consistency
147
+ if not h5_file.exists():
148
+ print(f" Copying H5 to cache...")
149
+ shutil.copy(local_h5, h5_file)
150
+ if not gtf_file.exists():
151
+ print(f" Copying GTF to cache...")
152
+ shutil.copy(local_gtf, gtf_file)
153
+ print()
154
+ print("=" * 70)
155
+ print("✅ Dataset ready!")
156
+ print("=" * 70)
157
+ print(f"H5 file: {h5_file}")
158
+ print(f"GTF file: {gtf_file}")
159
+ print(f"Cache location: {data_dir}")
160
+ print("=" * 70)
161
+ print()
162
+ return h5_file, gtf_file
163
+
164
+ # Download H5 file if needed
165
+ if not h5_file.exists() or force_download:
166
+ h5_url = "https://cf.10xgenomics.com/samples/cell-atac/2.0.0/atac_mouse_brain_5k_v1.1/atac_mouse_brain_5k_v1.1_filtered_peak_bc_matrix.h5"
167
+ try:
168
+ _download_file(h5_url, h5_file, "mouse brain 5k scATAC-seq H5 matrix")
169
+ except Exception as e:
170
+ print(f"\n⚠️ Automatic download failed: {e}")
171
+ print("\n" + "=" * 70)
172
+ print("MANUAL DOWNLOAD REQUIRED")
173
+ print("=" * 70)
174
+ print("Please download the file manually:")
175
+ print(f"\n1. Visit: https://www.10xgenomics.com/datasets/fresh-cortex-from-adult-mouse-brain-p-50-1-standard-2-0-0")
176
+ print(" Or try direct link:")
177
+ print(f" {h5_url}")
178
+ print(f"\n2. Save as: {h5_file}")
179
+ print(f"\n3. Or place in: {Path.cwd() / 'data' / 'mouse_brain_5k_v1.1.h5'}")
180
+ print("=" * 70)
181
+ raise
182
+ else:
183
+ print(f"✓ Using cached H5: {h5_file}")
184
+
185
+ # Download GTF if needed
186
+ if not gtf_file.exists() or force_download:
187
+ if not gtf_gz.exists() or force_download:
188
+ gtf_url = "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M25/gencode.vM25.annotation.gtf.gz"
189
+ _download_file(gtf_url, gtf_gz, "GENCODE vM25 mouse annotation (847 MB compressed)")
190
+
191
+ # Decompress
192
+ _decompress_gz(gtf_gz, gtf_file)
193
+
194
+ # Clean up compressed file to save space
195
+ if gtf_gz.exists():
196
+ gtf_gz.unlink()
197
+ print(f" ✓ Removed compressed file to save space")
198
+ else:
199
+ print(f"✓ Using cached GTF: {gtf_file}")
200
+
201
+ print()
202
+ print("=" * 70)
203
+ print("✅ Dataset ready!")
204
+ print("=" * 70)
205
+ print(f"H5 file: {h5_file}")
206
+ print(f"GTF file: {gtf_file}")
207
+ print(f"Cache location: {data_dir}")
208
+ print("=" * 70)
209
+ print()
210
+
211
+ return h5_file, gtf_file
212
+
213
+
214
+ def human_pbmc_5k_atacseq(force_download: bool = False) -> Tuple[Path, Path]:
215
+ """
216
+ Download 10X Human PBMC 5k scATAC-seq dataset and human annotation.
217
+
218
+ This dataset contains ~5,000 peripheral blood mononuclear cells (PBMCs),
219
+ profiled with 10X Chromium scATAC-seq. Automatically downloads both the
220
+ peak matrix (H5 format) and the human GENCODE annotation (GTF).
221
+
222
+ Parameters
223
+ ----------
224
+ force_download : bool, optional
225
+ If True, re-download even if files exist (default: False)
226
+
227
+ Returns
228
+ -------
229
+ h5_path : Path
230
+ Path to filtered_peak_bc_matrix.h5 file
231
+ gtf_path : Path
232
+ Path to gencode.v49.annotation.gtf file
233
+
234
+ Examples
235
+ --------
236
+ >>> import iaode
237
+ >>> h5_file, gtf_file = iaode.datasets.human_pbmc_5k_atacseq()
238
+ >>> adata = iaode.annotation_pipeline(h5_file, gtf_file)
239
+ """
240
+ data_dir = get_data_dir()
241
+ atacseq_dir = data_dir / "atacseq"
242
+ anno_dir = data_dir / "annotations"
243
+
244
+ # Define file paths
245
+ h5_file = atacseq_dir / "human_pbmc_5k_nextgem.h5"
246
+ gtf_file = anno_dir / "gencode.v49.annotation.gtf"
247
+ gtf_gz = anno_dir / "gencode.v49.annotation.gtf.gz"
248
+
249
+ # Download H5 file if needed
250
+ if not h5_file.exists() or force_download:
251
+ h5_url = "https://cf.10xgenomics.com/samples/cell-atac/2.0.0/atac_pbmc_5k_nextgem/atac_pbmc_5k_nextgem_filtered_peak_bc_matrix.h5"
252
+ _download_file(h5_url, h5_file, "human PBMC 5k scATAC-seq H5 matrix")
253
+ else:
254
+ print(f"✓ Using cached H5: {h5_file}")
255
+
256
+ # Download GTF if needed
257
+ if not gtf_file.exists() or force_download:
258
+ if not gtf_gz.exists() or force_download:
259
+ gtf_url = "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_49/gencode.v49.annotation.gtf.gz"
260
+ _download_file(gtf_url, gtf_gz, "GENCODE v49 human annotation (1.4 GB compressed)")
261
+
262
+ # Decompress
263
+ _decompress_gz(gtf_gz, gtf_file)
264
+
265
+ # Clean up compressed file
266
+ if gtf_gz.exists():
267
+ gtf_gz.unlink()
268
+ print(f" ✓ Removed compressed file to save space")
269
+ else:
270
+ print(f"✓ Using cached GTF: {gtf_file}")
271
+
272
+ print()
273
+ print("=" * 70)
274
+ print("✅ Dataset ready!")
275
+ print("=" * 70)
276
+ print(f"H5 file: {h5_file}")
277
+ print(f"GTF file: {gtf_file}")
278
+ print(f"Cache location: {data_dir}")
279
+ print("=" * 70)
280
+ print()
281
+
282
+ return h5_file, gtf_file
283
+
284
+
285
+ def clear_cache() -> None:
286
+ """
287
+ Clear the iAODE data cache directory.
288
+
289
+ Removes all downloaded datasets to free disk space.
290
+ Files will be re-downloaded on next use.
291
+ """
292
+ data_dir = get_data_dir()
293
+ if data_dir.exists():
294
+ shutil.rmtree(data_dir)
295
+ print(f"✓ Cleared cache: {data_dir}")
296
+ print(" Files will be re-downloaded when needed.")
297
+ else:
298
+ print("✓ Cache already empty")
299
+
300
+
301
+ def list_cached_files() -> None:
302
+ """
303
+ List all cached dataset files and their sizes.
304
+ """
305
+ data_dir = get_data_dir()
306
+ print("=" * 70)
307
+ print("iAODE Cached Datasets")
308
+ print("=" * 70)
309
+ print(f"Location: {data_dir}")
310
+ print()
311
+
312
+ total_size = 0
313
+ file_count = 0
314
+
315
+ for root, dirs, files in os.walk(data_dir):
316
+ for file in files:
317
+ filepath = Path(root) / file
318
+ size_mb = filepath.stat().st_size / 1024 / 1024
319
+ total_size += size_mb
320
+ file_count += 1
321
+ rel_path = filepath.relative_to(data_dir)
322
+ print(f" {rel_path}")
323
+ print(f" Size: {size_mb:.1f} MB")
324
+
325
+ print()
326
+ print(f"Total: {file_count} files, {total_size:.1f} MB")
327
+ print("=" * 70)
328
+
329
+ if file_count > 0:
330
+ print("\nTo clear cache: iaode.datasets.clear_cache()")
iaode/environment.py ADDED
@@ -0,0 +1,298 @@
1
+ # environment.py
2
+
3
+ from .model import iaodeModel
4
+ from .mixin import envMixin
5
+ import numpy as np
6
+ from sklearn.cluster import KMeans
7
+ from sklearn.preprocessing import LabelEncoder
8
+ import torch
9
+ from torch.utils.data import DataLoader, TensorDataset
10
+
11
+
12
+ class Env(iaodeModel, envMixin):
13
+ """
14
+ Training environment for iAODE model.
15
+
16
+ Handles data registration, train/val/test splitting, DataLoader creation,
17
+ epoch-based training, validation, and early stopping.
18
+
19
+ Parameters
20
+ ----------
21
+ adata : AnnData
22
+ Annotated data object
23
+ layer : str
24
+ Layer name for count data
25
+ train_size : float
26
+ Training set fraction
27
+ val_size : float
28
+ Validation set fraction
29
+ test_size : float
30
+ Test set fraction
31
+ batch_size : int
32
+ Batch size for training
33
+ random_seed : int
34
+ Random seed for reproducibility
35
+ **kwargs : dict
36
+ Additional parameters passed to iaodeModel
37
+ """
38
+
39
+ def __init__(
40
+ self,
41
+ adata,
42
+ layer,
43
+ recon,
44
+ irecon,
45
+ beta,
46
+ dip,
47
+ tc,
48
+ info,
49
+ hidden_dim,
50
+ latent_dim,
51
+ i_dim,
52
+ use_ode,
53
+ loss_mode,
54
+ lr,
55
+ vae_reg,
56
+ ode_reg,
57
+ device,
58
+ train_size,
59
+ val_size,
60
+ test_size,
61
+ batch_size,
62
+ random_seed,
63
+ encoder_type,
64
+ encoder_num_layers,
65
+ encoder_n_heads,
66
+ encoder_d_model,
67
+ *args,
68
+ **kwargs,
69
+ ):
70
+ # Store split parameters
71
+ self.train_size = train_size
72
+ self.val_size = val_size
73
+ self.test_size = test_size
74
+ self.batch_size_fixed = batch_size
75
+ self.random_seed = random_seed
76
+
77
+ # Register data and create splits
78
+ self._register_anndata(adata, layer, latent_dim)
79
+
80
+ # Initialize model
81
+ super().__init__(
82
+ recon=recon,
83
+ irecon=irecon,
84
+ beta=beta,
85
+ dip=dip,
86
+ tc=tc,
87
+ info=info,
88
+ state_dim=self.n_var,
89
+ hidden_dim=hidden_dim,
90
+ latent_dim=latent_dim,
91
+ i_dim=i_dim,
92
+ use_ode=use_ode,
93
+ loss_mode=loss_mode,
94
+ lr=lr,
95
+ vae_reg=vae_reg,
96
+ ode_reg=ode_reg,
97
+ device=device,
98
+ encoder_type=encoder_type,
99
+ encoder_num_layers=encoder_num_layers,
100
+ encoder_n_heads=encoder_n_heads,
101
+ encoder_d_model=encoder_d_model,
102
+ *args,
103
+ **kwargs,
104
+ )
105
+
106
+ # Training tracking
107
+ self.score = []
108
+ self.train_losses = []
109
+ self.val_losses = []
110
+ self.val_scores = []
111
+
112
+ # Early stopping state
113
+ self.best_val_loss = float('inf')
114
+ self.best_model_state = None
115
+ self.patience_counter = 0
116
+
117
+ def _register_anndata(self, adata, layer: str, latent_dim: int):
118
+ """Load data and create train/val/test splits"""
119
+
120
+ # Load and log-transform data
121
+ if hasattr(adata.layers[layer], 'toarray'):
122
+ self.X = np.log1p(adata.layers[layer].toarray())
123
+ else:
124
+ self.X = np.log1p(adata.layers[layer])
125
+
126
+ self.n_obs = adata.shape[0]
127
+ self.n_var = adata.shape[1]
128
+
129
+ # Generate evaluation labels
130
+ if 'cell_type' in adata.obs.columns:
131
+ self.labels = LabelEncoder().fit_transform(adata.obs['cell_type'])
132
+ else:
133
+ # Cast to ndarray to ensure downstream indexing works without type warnings
134
+ self.labels = np.asarray(
135
+ KMeans(latent_dim, random_state=self.random_seed).fit_predict(self.X)
136
+ )
137
+
138
+ # Create train/val/test splits
139
+ np.random.seed(self.random_seed)
140
+ indices = np.random.permutation(self.n_obs)
141
+
142
+ n_train = int(self.train_size * self.n_obs)
143
+ n_val = int(self.val_size * self.n_obs)
144
+
145
+ self.train_idx = indices[:n_train]
146
+ self.val_idx = indices[n_train:n_train + n_val]
147
+ self.test_idx = indices[n_train + n_val:]
148
+
149
+ # Split data
150
+ self.X_train = self.X[self.train_idx]
151
+ self.X_val = self.X[self.val_idx]
152
+ self.X_test = self.X[self.test_idx]
153
+
154
+ # Indexing warnings suppressed – runtime types are ndarray
155
+ self.labels_train = self.labels[self.train_idx] # type: ignore[index]
156
+ self.labels_val = self.labels[self.val_idx] # type: ignore[index]
157
+ self.labels_test = self.labels[self.test_idx] # type: ignore[index]
158
+
159
+ print("\n" + "="*70)
160
+ print("Data Split")
161
+ print("="*70)
162
+ print(f" Train: {len(self.train_idx):,} cells ({len(self.train_idx)/self.n_obs*100:.1f}%)")
163
+ print(f" Val: {len(self.val_idx):,} cells ({len(self.val_idx)/self.n_obs*100:.1f}%)")
164
+ print(f" Test: {len(self.test_idx):,} cells ({len(self.test_idx)/self.n_obs*100:.1f}%)")
165
+
166
+ # Create DataLoaders
167
+ self._create_dataloaders()
168
+
169
+ def _create_dataloaders(self):
170
+ """Create PyTorch DataLoaders for training, validation, and testing"""
171
+
172
+ # Convert to tensors
173
+ X_train_tensor = torch.FloatTensor(self.X_train)
174
+ X_val_tensor = torch.FloatTensor(self.X_val)
175
+ X_test_tensor = torch.FloatTensor(self.X_test)
176
+
177
+ # Create datasets
178
+ train_dataset = TensorDataset(X_train_tensor)
179
+ val_dataset = TensorDataset(X_val_tensor)
180
+ test_dataset = TensorDataset(X_test_tensor)
181
+
182
+ # Create dataloaders
183
+ self.train_loader = DataLoader(
184
+ train_dataset,
185
+ batch_size=self.batch_size_fixed,
186
+ shuffle=True,
187
+ drop_last=True
188
+ )
189
+
190
+ self.val_loader = DataLoader(
191
+ val_dataset,
192
+ batch_size=self.batch_size_fixed,
193
+ shuffle=False,
194
+ drop_last=False
195
+ )
196
+
197
+ self.test_loader = DataLoader(
198
+ test_dataset,
199
+ batch_size=self.batch_size_fixed,
200
+ shuffle=False,
201
+ drop_last=False
202
+ )
203
+
204
+ print(f" Batches per epoch: {len(self.train_loader)}")
205
+ print("="*70)
206
+
207
+ def train_epoch(self):
208
+ """Train for one complete epoch"""
209
+
210
+ self.train()
211
+ epoch_losses = []
212
+
213
+ for batch_data, in self.train_loader:
214
+ batch_data = batch_data.to(self.device)
215
+ self.update(batch_data)
216
+ epoch_losses.append(self.loss[-1][0])
217
+
218
+ avg_train_loss = np.mean(epoch_losses)
219
+ self.train_losses.append(avg_train_loss)
220
+
221
+ return avg_train_loss
222
+
223
+ def validate(self):
224
+ """Evaluate on validation set"""
225
+
226
+ self.eval()
227
+ val_losses = []
228
+ all_latents = []
229
+
230
+ with torch.no_grad():
231
+ for batch_data, in self.val_loader:
232
+ batch_data = batch_data.to(self.device)
233
+
234
+ # Compute loss
235
+ loss_value = self._compute_loss_only(batch_data)
236
+ val_losses.append(loss_value)
237
+
238
+ # Extract latent representations
239
+ latent = self.take_latent(batch_data)
240
+ all_latents.append(latent)
241
+
242
+ # Average validation loss
243
+ avg_val_loss = np.mean(val_losses)
244
+ self.val_losses.append(avg_val_loss)
245
+
246
+ # Compute clustering metrics
247
+ all_latents = np.concatenate(all_latents, axis=0)
248
+ val_score = self._calc_score_with_labels(all_latents, self.labels_val)
249
+ self.val_scores.append(val_score)
250
+
251
+ return avg_val_loss, val_score
252
+
253
+ def check_early_stopping(self, val_loss: float, patience: int = 20):
254
+ """
255
+ Check early stopping criterion.
256
+
257
+ Parameters
258
+ ----------
259
+ val_loss : float
260
+ Current validation loss
261
+ patience : int, default=20
262
+ Number of epochs without improvement before stopping
263
+
264
+ Returns
265
+ -------
266
+ should_stop : bool
267
+ Whether training should stop
268
+ improved : bool
269
+ Whether validation loss improved
270
+ """
271
+
272
+ if val_loss < self.best_val_loss:
273
+ # Save best model
274
+ self.best_val_loss = val_loss
275
+ self.best_model_state = {
276
+ k: v.cpu().clone() for k, v in self.state_dict().items()
277
+ }
278
+ self.patience_counter = 0
279
+ return False, True
280
+ else:
281
+ # Increment patience counter
282
+ self.patience_counter += 1
283
+
284
+ if self.patience_counter >= patience:
285
+ return True, False
286
+ else:
287
+ return False, False
288
+
289
+ def load_best_model(self):
290
+ """Restore best model from early stopping checkpoint"""
291
+
292
+ if self.best_model_state is not None:
293
+ self.load_state_dict(self.best_model_state)
294
+ print(f"\n{'='*70}")
295
+ print(f"Loaded best model (val_loss={self.best_val_loss:.4f})")
296
+ print(f"{'='*70}\n")
297
+ else:
298
+ print("\nWarning: No saved model state found!")