panhumanpy 0.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. panhumanpy/ANNotate.py +2642 -0
  2. panhumanpy/ANNotate_tools.py +3931 -0
  3. panhumanpy/__init__.py +27 -0
  4. panhumanpy/_tools/__init__.py +3 -0
  5. panhumanpy/_tools/v0/__init__.py +14 -0
  6. panhumanpy/_tools/v0/calibration/__init__.py +0 -0
  7. panhumanpy/_tools/v0/cell_ontology_map/__init__.py +0 -0
  8. panhumanpy/_tools/v0/cell_ontology_map/cell_ontology_map.csv +447 -0
  9. panhumanpy/_tools/v0/inference_encoders/__init__.py +0 -0
  10. panhumanpy/_tools/v0/inference_encoders/inference_encoders.pkl +0 -0
  11. panhumanpy/_tools/v0/inference_feature_panel/__init__.py +0 -0
  12. panhumanpy/_tools/v0/inference_feature_panel/inference_feature_panel.txt +5039 -0
  13. panhumanpy/_tools/v0/inference_model/__init__.py +0 -0
  14. panhumanpy/_tools/v0/postprocessing/__init__.py +0 -0
  15. panhumanpy/_tools/v0/postprocessing/panhuman_annotate_broad.csv +393 -0
  16. panhumanpy/_tools/v0/postprocessing/panhuman_annotate_fine.csv +398 -0
  17. panhumanpy/_tools/v0/postprocessing/panhuman_annotate_medium.csv +393 -0
  18. panhumanpy/_tools/v1/__init__.py +14 -0
  19. panhumanpy/_tools/v1/calibration/__init__.py +0 -0
  20. panhumanpy/_tools/v1/calibration/custom_objects.py +151 -0
  21. panhumanpy/_tools/v1/calibration/temp_scaler_L_1.keras +0 -0
  22. panhumanpy/_tools/v1/calibration/temp_scaler_L_2.keras +0 -0
  23. panhumanpy/_tools/v1/calibration/temp_scaler_L_3.keras +0 -0
  24. panhumanpy/_tools/v1/calibration/temp_scaler_L_4.keras +0 -0
  25. panhumanpy/_tools/v1/calibration/temp_scaler_L_5.keras +0 -0
  26. panhumanpy/_tools/v1/calibration/temp_scaler_L_6.keras +0 -0
  27. panhumanpy/_tools/v1/calibration/temp_scaler_L_7.keras +0 -0
  28. panhumanpy/_tools/v1/calibration/temp_scaler_L_8.keras +0 -0
  29. panhumanpy/_tools/v1/cell_ontology_map/__init__.py +0 -0
  30. panhumanpy/_tools/v1/cell_ontology_map/cell_ontology_map.csv +447 -0
  31. panhumanpy/_tools/v1/inference_encoders/__init__.py +0 -0
  32. panhumanpy/_tools/v1/inference_encoders/inference_encoders.pkl +0 -0
  33. panhumanpy/_tools/v1/inference_feature_panel/__init__.py +0 -0
  34. panhumanpy/_tools/v1/inference_feature_panel/inference_feature_panel.txt +5055 -0
  35. panhumanpy/_tools/v1/inference_model/__init__.py +0 -0
  36. panhumanpy/_tools/v1/postprocessing/__init__.py +0 -0
  37. panhumanpy/_tools/v1/postprocessing/panhuman_annotate_broad.csv +395 -0
  38. panhumanpy/_tools/v1/postprocessing/panhuman_annotate_fine.csv +399 -0
  39. panhumanpy/_tools/v1/postprocessing/panhuman_annotate_medium.csv +395 -0
  40. panhumanpy/loss_fn.py +40 -0
  41. panhumanpy-0.5.0.dist-info/METADATA +95 -0
  42. panhumanpy-0.5.0.dist-info/RECORD +44 -0
  43. panhumanpy-0.5.0.dist-info/WHEEL +4 -0
  44. panhumanpy-0.5.0.dist-info/entry_points.txt +2 -0
panhumanpy/ANNotate.py ADDED
@@ -0,0 +1,2642 @@
1
+ """
2
+ Azimuth Cell Annotation: Neural network-based hierarchical cell type
3
+ annotation for single-cell RNA-seq data.
4
+
5
+ This module provides tools for hierarchical cell type annotation and
6
+ interpretation based on single-cell RNA-seq data using the Azimuth
7
+ neural network model trained on annotated panhuman scRNA-seq data.
8
+
9
+ Key Components
10
+ -------------
11
+ AzimuthNN_base : Class
12
+ Low-level class providing fine-grained control over the annotation
13
+ process. Suitable for advanced users who need detailed control or
14
+ are processing data in batches to optimize memory usage.
15
+
16
+ AzimuthNN : Class
17
+ High-level interface that wraps around AzimuthNN_base for interactive
18
+ analysis. Provides a streamlined workflow for cell annotation with
19
+ sensible defaults. Recommended for most interactive analysis sessions
20
+ and notebooks.
21
+
22
+ annotate_core : Function
23
+ Core function for script-based automated annotation. Designed for
24
+ batch processing and integration into analysis pipelines.
25
+
26
+ Usage Examples
27
+ -------------
28
+ Interactive usage with high-level interface:
29
+ >>> import anndata
30
+ >>> from panhumanpy import AzimuthNN
31
+ >>> adata = anndata.read_h5ad('my_data.h5ad')
32
+ >>> # Run minimal annotation pipeline with calibration
33
+ >>> azimuth = AzimuthNN(adata)
34
+ >>> embeddings = azimuth.azimuth_embed() # Extract embeddings
35
+ >>> umap = azimuth.azimuth_umap() # Generate UMAP
36
+ >>> # Optionally map annotations to Cell Ontology terms
37
+ >>> azimuth.map_to_cell_ontology('azimuth_fine')
38
+ >>> azimuth.map_to_cell_ontology('azimuth_broad', include_cl_id=True)
39
+ >>> adata_annotated = azimuth.pack_adata('output.h5ad') # Save results
40
+
41
+ For more detailed documentation on specific classes and functions:
42
+ >>> help(AzimuthNN)
43
+ >>> help(AzimuthNN_base)
44
+ >>> help(annotate_core)
45
+
46
+ Command-line Usage
47
+ -----------------
48
+ This module can be run as a standalone script to annotate h5ad files:
49
+
50
+ annotate /path/to/input.h5ad [options]
51
+
52
+ Required positional argument:
53
+ filepath Path to input h5ad file containing
54
+ single-cell data
55
+
56
+ Optional arguments:
57
+ -fn, --feature_names_col
58
+ Column in query.var containing gene
59
+ names (default: None)
60
+ -ap, --annotation_pipeline
61
+ Annotation pipeline to use
62
+ (default: 'supervised')
63
+ -ebs, --eval_batch_size
64
+ Batch size for model inference
65
+ (default: 8192)
66
+ -norm, --normalization_override
67
+ Skip normalization check (default: False)
68
+ -ncbs, --norm_check_batch_size
69
+ Number of cells to sample for normalization
70
+ check (default: 100)
71
+ -om, --output_mode Output verbosity, 'minimal' or 'detailed'
72
+ (default: 'minimal')
73
+ -rf, --refine_labels Skip hierarchical label refinement
74
+ (default: use refinement)
75
+ -mcl, --map_to_cl One or more column names in the cell metadata
76
+ to map to Cell Ontology labels after annotation.
77
+ Multiple columns can be provided as a
78
+ space-separated list.
79
+ e.g. -mcl azimuth_broad azimuth_fine
80
+ (default: None, no mapping applied)
81
+ -clid, --include_cl_id
82
+ If set, also adds CL ID columns
83
+ (e.g. CL:0000236) alongside CL label columns
84
+ produced by --map_to_cl. Has no effect if
85
+ --map_to_cl is not specified.
86
+ (default: False)
87
+ -em, --extract_embeddings
88
+ Skip neural network embeddings extraction
89
+ (default: extract embeddings)
90
+ -umap, --umap_embeddings
91
+ Skip UMAP projection generation
92
+ (default: generate UMAP)
93
+
94
+ UMAP parameters:
95
+ -nnbrs, --n_neighbors Neighbors per point in UMAP (default: 30)
96
+ -nc, --n_components UMAP dimensionality (default: 2)
97
+ -me, --metric Distance metric for UMAP (default: 'cosine')
98
+ -mdt, --min_dist Minimum distance in UMAP (default: 0.3)
99
+ -ulr, --umap_lr UMAP learning rate (default: 1.0)
100
+ -useed, --umap_seed Random seed for reproducibility (default: 42)
101
+ -sp, --spread UMAP spread parameter (default: 1.0)
102
+ -uv, --umap_verbose Hide UMAP progress
103
+ (default: show progress)
104
+ -uin, --umap_init UMAP initialization method (default: 'spectral')
105
+
106
+ Output:
107
+ The annotated data will be saved as a new h5ad file in the same directory
108
+ as the input file, with '_ANN' appended to the filename. If a file with
109
+ that name already exists, a timestamp (YYYYMMDD_HHMMSS) will be
110
+ automatically appended to prevent overwriting existing results.
111
+
112
+ Example commands:
113
+ annotate my_cells.h5ad -fn feature_name -ebs 4096 -nc 3
114
+ annotate my_cells.h5ad -mcl azimuth_broad azimuth_fine -clid
115
+ """
116
+
117
+
118
+
119
+ from panhumanpy.ANNotate_tools import *
120
+
121
+ model_version_default = 'v0'
122
+
123
+ _gpu_configured = False
124
+
125
+ def configure_once():
126
+ """
127
+ Configures TensorFlow GPU settings once per process.
128
+ This wrapper ensures the configuration only happens once.
129
+ """
130
+ global _gpu_configured
131
+ if not _gpu_configured:
132
+ configure()
133
+ _gpu_configured = True
134
+ return True
135
+ return False
136
+
137
+ configure_once()
138
+
139
+
140
+ ########################################################################
141
+ ###### Base class for low level interactive usage ######################
142
+
143
+
144
+ class AzimuthNN_base(AutoloadInferenceTools):
145
+ """
146
+ Base class for low-level interactive usage of the Azimuth neural
147
+ network annotation pipeline.
148
+
149
+ This class provides a comprehensive framework for single-cell
150
+ RNA-seq annotation using neural network models. It handles the
151
+ complete workflow from data loading and preprocessing to inference,
152
+ confidence calibration, post-processing, and result visualization.
153
+
154
+ This includes functionality for extracting embeddings, generating
155
+ UMAP visualizations, and refining annotations at different levels of
156
+ granularity.
157
+
158
+ Parameters
159
+ ----------
160
+ annotation_pipeline : str, default='supervised'
161
+ The type of annotation pipeline to use.
162
+ model_version: str, default set to match package major version.
163
+ Model version to use.
164
+ eval_batch_size : int, default=8192
165
+ Batch size for inference and embedding generation.
166
+
167
+ Attributes
168
+ ----------
169
+ query : anndata.AnnData or None
170
+ The AnnData object if loaded.
171
+ X_query : scipy.sparse.csr_matrix or None
172
+ Expression matrix in CSR format.
173
+ query_features : list or None
174
+ List of feature names.
175
+ features_meta : pandas.DataFrame or None
176
+ Feature metadata.
177
+ cells_meta : pandas.DataFrame or None
178
+ Cell metadata.
179
+ num_cells : int or None
180
+ Number of cells in the query.
181
+ processed_outputs : dict or None
182
+ Processed inference results.
183
+ embeddings : dict
184
+ Dictionary of extracted embeddings.
185
+ umaps : dict
186
+ Dictionary of generated UMAP coordinates.
187
+
188
+ Raises
189
+ ------
190
+ TypeError
191
+ If input parameters are not of the correct type.
192
+ RuntimeError
193
+ If model metadata fails to load.
194
+
195
+ Notes
196
+ -----
197
+ This class is designed for programmatic use and provides
198
+ fine-grained control over each step of the annotation pipeline.
199
+ Consider using a higher-level interface for convenience if a
200
+ standard workflow is sufficient for your needs.
201
+ """
202
+ def __init__(
203
+ self,
204
+ annotation_pipeline='supervised',
205
+ model_version=model_version_default,
206
+ eval_batch_size=8192,
207
+ ):
208
+
209
+ if not isinstance(annotation_pipeline, str):
210
+ raise TypeError("annotation_pipeline must be a string")
211
+
212
+ if not isinstance(model_version, str):
213
+ raise TypeError("model_version must be a string")
214
+
215
+ if not isinstance(eval_batch_size, int):
216
+ raise TypeError("eval_batch_size must be an integer")
217
+
218
+
219
+
220
+ self._annotation_pipeline = annotation_pipeline
221
+ self._model_version = model_version
222
+ self._eval_batch_size = eval_batch_size
223
+ self._verbose = True
224
+
225
+
226
+ super().__init__(annotation_pipeline, model_version)
227
+
228
+ if not hasattr(self, 'model_meta'):
229
+ raise RuntimeError("Failed to load model metadata")
230
+
231
+ for meta_key in self.model_meta.keys():
232
+ if not isinstance(meta_key, str):
233
+ raise TypeError("All model metadata keys must be strings")
234
+ setattr(self, meta_key, self.model_meta[meta_key])
235
+
236
+ self.query = None
237
+ self.X_query = None
238
+ self.query_features = None
239
+ self.features_meta = None
240
+ self.cells_meta = None
241
+ self.num_cells = None
242
+ self._inference_input_matrix = None
243
+
244
+ self._inference_outputs_unprocessed = None
245
+ self.processed_outputs = None
246
+ self._azimuth_refined_labels = {}
247
+ self.embeddings = {}
248
+ self.umaps = {}
249
+
250
+
251
+
252
+ def query_stripped(
253
+ self,
254
+ X_query,
255
+ query_features,
256
+ cells_meta
257
+ ):
258
+ """
259
+ Load query data directly from expression matrix and metadata.
260
+
261
+ This method allows for direct loading of pre-processed
262
+ expression data without requiring an AnnData object. This is
263
+ useful for integration with custom preprocessing pipelines.
264
+
265
+ Parameters
266
+ ----------
267
+ X_query : scipy.sparse.csr_matrix
268
+ Expression matrix with cells as rows and features as columns.
269
+ query_features : list of str
270
+ List of feature names corresponding to columns in X_query.
271
+ cells_meta : pandas.DataFrame
272
+ Cell metadata with rows corresponding to cells in X_query.
273
+
274
+ Raises
275
+ ------
276
+ TypeError
277
+ If inputs are not of correct type.
278
+ ValueError
279
+ If dimensions of inputs don't match.
280
+
281
+ Notes
282
+ -----
283
+ This method creates a minimal features_meta DataFrame based on
284
+ the provided feature names.
285
+ """
286
+ if not isinstance(X_query, csr_matrix):
287
+ raise TypeError("X_query must be a scipy.sparse.csr_matrix")
288
+
289
+ if not isinstance(query_features, list) or not all(isinstance(f, str)
290
+ for f in query_features):
291
+ raise TypeError("query_features must be a list of strings")
292
+
293
+ if not isinstance(cells_meta, pd.DataFrame):
294
+ raise TypeError("cells_meta must be a pandas DataFrame")
295
+
296
+ if len(query_features) != X_query.shape[1]:
297
+ raise ValueError(
298
+ f"Number of features ({len(query_features)}) "
299
+ f"does not match X_query columns ({X_query.shape[1]})"
300
+ )
301
+
302
+ if cells_meta.shape[0] != X_query.shape[0]:
303
+ raise ValueError(
304
+ f"Number of cells in metadata "
305
+ f"({cells_meta.shape[0]}) does not match X_query rows "
306
+ f"({X_query.shape[0]})"
307
+ )
308
+
309
+ if X_query.dtype != np.float64:
310
+ X_query = X_query.astype(np.float64)
311
+
312
+ self.X_query = X_query
313
+ self.query_features = query_features
314
+ self.features_meta = pd.DataFrame(
315
+ {'feature_name':query_features},
316
+ index = query_features
317
+ )
318
+ self.cells_meta = cells_meta
319
+ self.num_cells = X_query.shape[0]
320
+
321
+
322
+ def query_adata(
323
+ self,
324
+ query_arg,
325
+ feature_names_col=None
326
+ ):
327
+ """
328
+ Load query data from an AnnData object.
329
+
330
+ Parameters
331
+ ----------
332
+ query_arg : anndata.AnnData
333
+ AnnData object containing expression data and metadata.
334
+ feature_names_col : str, optional
335
+ Column in var DataFrame to use for feature names.
336
+ If None, uses the var_names index.
337
+
338
+ Notes
339
+ -----
340
+ This method extracts the expression matrix, feature names,
341
+ and metadata from the provided AnnData object.
342
+ """
343
+ query_obj = QueryObj(query_arg)
344
+
345
+ self.X_query = query_obj.X_query()
346
+ if self.X_query.dtype != np.float64:
347
+ self.X_query = self.X_query.astype(np.float64)
348
+
349
+ self.query_features = query_obj.query_features(
350
+ feature_names_col=feature_names_col
351
+ )
352
+ self.features_meta = query_obj.features_meta()
353
+ self.cells_meta = query_obj.cells_meta()
354
+ self.num_cells = self.X_query.shape[0]
355
+
356
+ def query_h5ad(
357
+ self,
358
+ query_filepath,
359
+ feature_names_col=None
360
+ ):
361
+ """
362
+ Load query data from an H5AD file on disk.
363
+
364
+ Parameters
365
+ ----------
366
+ query_filepath : str
367
+ Path to the H5AD file containing the query data.
368
+ feature_names_col : str, optional
369
+ Column in var DataFrame to use for feature names.
370
+ If None, uses the var_names index.
371
+
372
+ Raises
373
+ ------
374
+ ValueError
375
+ If the file is not in H5AD format.
376
+
377
+ Notes
378
+ -----
379
+ This method reads the H5AD file from disk and extracts the
380
+ necessary components for inference.
381
+ """
382
+ query_obj = ReadQueryObj(query_filepath)
383
+
384
+ self.X_query = query_obj.X_query()
385
+ if self.X_query.dtype != np.float64:
386
+ self.X_query = self.X_query.astype(np.float64)
387
+
388
+ self.query_features = query_obj.query_features(
389
+ feature_names_col=feature_names_col
390
+ )
391
+ self.features_meta = query_obj.features_meta()
392
+ self.cells_meta = query_obj.cells_meta()
393
+ self.num_cells = self.X_query.shape[0]
394
+
395
+ def process_query(
396
+ self,
397
+ normalization_override=False,
398
+ norm_check_batch_size=100
399
+ ):
400
+ """
401
+ Process the query data to prepare it for inference.
402
+
403
+ This method prepares the expression data for the inference model
404
+ according to the specified annotation pipeline. The processing steps
405
+ vary depending on the pipeline type, potentially including
406
+ normalization, feature selection, dimensionality reduction, or
407
+ other transformations.
408
+
409
+ Parameters
410
+ ----------
411
+ normalization_override : bool, default=False
412
+ If True, bypasses normalization entirely regardless of
413
+ whether the values are integers or not.
414
+ norm_check_batch_size : int, default=100
415
+ Batch size for checking normalization status.
416
+
417
+ Raises
418
+ ------
419
+ TypeError
420
+ If parameters are not of the correct type.
421
+
422
+ Notes
423
+ -----
424
+ This method must be called after loading query data and before
425
+ running inference or extracting embeddings. The specific processing
426
+ steps depend on the annotation_pipeline specified during
427
+ initialization.
428
+
429
+ Currently, only the 'supervised' annotation pipeline is implemented,
430
+ which normalizes the expression data and aligns it with a reference
431
+ feature panel.
432
+ """
433
+
434
+ if not isinstance(normalization_override, bool):
435
+ raise TypeError("normalization override must be a bool")
436
+
437
+ if not isinstance(norm_check_batch_size, int):
438
+ raise TypeError("norm_check_batch_size must be an integer")
439
+
440
+ query_processing_class = InferenceInputData(
441
+ self.X_query,
442
+ self.query_features,
443
+ self.inference_feature_panel,
444
+ normalization_override = normalization_override,
445
+ norm_check_batch_size = norm_check_batch_size
446
+ )
447
+
448
+ self._inference_input_matrix = query_processing_class.inference_input(
449
+ annotation_pipeline = self._annotation_pipeline
450
+ )
451
+
452
+
453
+ def run_inference_model(self):
454
+ """
455
+ Run the inference model on the processed query data.
456
+
457
+ This method executes the neural network inference to generate
458
+ cell type predictions.
459
+
460
+ Returns
461
+ -------
462
+ dict
463
+ Dictionary of raw inference outputs including hierarchical
464
+ label predictions and probabilities.
465
+
466
+ Raises
467
+ ------
468
+ AssertionError
469
+ If input matrix has not been initialized by calling
470
+ process_query().
471
+
472
+ Notes
473
+ -----
474
+ The raw outputs should be calibrated using calibrate_predictions()
475
+ and then processed using process_outputs() before further use
476
+ downstream. The typical workflow is:
477
+ 1. run_inference_model()
478
+ 2. calibrate_predictions()
479
+ 3. process_outputs()
480
+ """
481
+
482
+ assert self._inference_input_matrix is not None, (
483
+ "Input matrix not initialized. Call process_query() first."
484
+ )
485
+
486
+ inference_class = Inference(
487
+ self._inference_input_matrix,
488
+ self.inference_model,
489
+ self.inference_encoders,
490
+ self._eval_batch_size,
491
+ self.max_depth,
492
+ verbose = self._verbose
493
+ )
494
+
495
+ self._inference_outputs_unprocessed = inference_class.run_inference()
496
+
497
+ return self._inference_outputs_unprocessed
498
+
499
+ def calibrate_predictions(self):
500
+ """
501
+ Apply calibration to softmax outputs using trained calibrators.
502
+
503
+ This method calibrates the softmax probability outputs from each
504
+ hierarchical level using the corresponding trained calibration models,
505
+ if available. Calibration improves the reliability and trustworthiness
506
+ of prediction confidence scores by adjusting for overconfidence or
507
+ underconfidence in the original model outputs.
508
+
509
+ Returns
510
+ -------
511
+ dict
512
+ Updated inference outputs dictionary with calibrated results.
513
+ Contains the same keys as the original inference outputs but with
514
+ calibrated values:
515
+
516
+ - 'softmax_vals_all': List of calibrated softmax probability arrays,
517
+ one per hierarchical level
518
+ - 'probability_of_preds': Updated maximum probability values from
519
+ the calibrated softmax distributions
520
+ - Other keys remain unchanged from the original inference outputs
521
+
522
+ Notes
523
+ -----
524
+ - Only applies calibration if calibration method is specified in model
525
+ metadata and calibrator models are available
526
+ - If no calibration is configured (calibration method is None), the
527
+ method returns the original inference outputs unchanged
528
+ - Each hierarchical level is calibrated independently using its own
529
+ trained calibrator model
530
+ - Memory management is applied during processing to handle large datasets
531
+ efficiently by cleaning up intermediate results after each level
532
+
533
+ Raises
534
+ ------
535
+ AssertionError
536
+ If calibration method is not None but the number of available
537
+ calibrator models doesn't match the expected number of hierarchical
538
+ levels (max_depth).
539
+
540
+ Examples
541
+ --------
542
+ The method is typically called as part of the inference pipeline:
543
+
544
+ >>> azimuth = AzimuthNN_base()
545
+ >>> # ... load data and run inference ...
546
+ >>> raw_outputs = azimuth.run_inference_model()
547
+ >>> calibrated_outputs = azimuth.calibrate_predictions()
548
+ >>> # Calibrated outputs now have adjusted confidence scores
549
+
550
+ The calibration process transforms prediction confidence scores:
551
+
552
+ - Before calibration: Model might be over(/under)-confident
553
+ (high probabilities for uncertain predictions or vice-versa)
554
+ - After calibration: Probabilities better reflect true prediction
555
+ confidence and uncertainty
556
+
557
+ """
558
+
559
+ calibration_method = self.model_meta['calibration']
560
+ if calibration_method is not None:
561
+ assert len(self.calibrators)==self.max_depth, (
562
+ "Calibration method is not None, expected number of "
563
+ f"calibrator models: {self.max_depth}, number of "
564
+ f"calibrator models found: {len(self.calibrators)}"
565
+ )
566
+
567
+ softmax_all = self._inference_outputs_unprocessed[
568
+ 'softmax_vals_all'
569
+ ]
570
+
571
+ calibrated_levels_cache = []
572
+ max_probs_levels_cache = []
573
+
574
+ for level in range(self.max_depth):
575
+ with MemoryContext():
576
+ sm_array = softmax_all[level]
577
+ calibrator_model = self.calibrators[level]
578
+
579
+
580
+ calibration_obj = CalibrationSingleClassifier(
581
+ softmax = sm_array,
582
+ eval_batch_size = self._eval_batch_size,
583
+ verbose = self._verbose
584
+ )
585
+
586
+ calibrated_sm = calibration_obj.calibrate(
587
+ calibration_method,
588
+ calibrator_model
589
+ )
590
+
591
+ calibrated_levels_cache.append(calibrated_sm)
592
+ max_probs_levels_cache.append(
593
+ np.max(calibrated_sm, axis=-1)
594
+ )
595
+
596
+ max_probs = np.column_stack(max_probs_levels_cache)
597
+
598
+ self._inference_outputs_unprocessed[
599
+ 'softmax_vals_all'
600
+ ] = calibrated_levels_cache
601
+
602
+ self._inference_outputs_unprocessed[
603
+ 'probability_of_preds'
604
+ ] = max_probs
605
+
606
+ return self._inference_outputs_unprocessed
607
+
608
+
609
+
610
+ def process_outputs(self, mode='minimal'):
611
+ """
612
+ Process raw inference outputs into usable predictions.
613
+
614
+ This method organizes the raw inference outputs into a structured
615
+ dictionary of predictions at various hierarchical levels.
616
+
617
+ Parameters
618
+ ----------
619
+ mode : str, default='minimal'
620
+ Processing mode: 'minimal' provides essential outputs,
621
+ 'detailed' includes additional information for all levels.
622
+
623
+ Returns
624
+ -------
625
+ dict
626
+ Dictionary of processed outputs including hierarchical labels,
627
+ level-specific labels, and confidence scores.
628
+
629
+ Raises
630
+ ------
631
+ AssertionError
632
+ If mode is not 'minimal' or 'detailed'.
633
+
634
+ Notes
635
+ -----
636
+ This method should be called after run_inference_model() and
637
+ calibrate_predictions(). The calibration step improves confidence
638
+ score reliability by correcting for model overconfidence or
639
+ underconfidence using trained calibration models.
640
+ """
641
+
642
+ assert mode in ['minimal','detailed'], (
643
+ "mode for output processing should be either "
644
+ "'minimal' or 'detailed'"
645
+ )
646
+
647
+ labels_pred = self._inference_outputs_unprocessed[
648
+ 'hierarchical_label_preds'
649
+ ]
650
+ labels_prob = self._inference_outputs_unprocessed[
651
+ 'probability_of_preds'
652
+ ]
653
+
654
+ output_processing_class = OutputLabels(
655
+ labels_pred,
656
+ labels_prob,
657
+ self.max_depth,
658
+ self.num_cells
659
+ )
660
+
661
+ combined_labels = output_processing_class.combined_labels
662
+ level_zero_labels = output_processing_class.level_zero_labels
663
+ final_level_labels = output_processing_class.final_level_labels
664
+ final_level_prob = (
665
+ output_processing_class.final_level_prob
666
+ )
667
+ full_consistent_hierarchy = (
668
+ output_processing_class.full_consistent_hierarchy
669
+ )
670
+
671
+ self.processed_outputs = {
672
+ 'full_hierarchical_labels': combined_labels,
673
+ 'level_zero_labels': level_zero_labels,
674
+ 'final_level_labels': final_level_labels,
675
+ 'final_level_confidence': final_level_prob,
676
+ 'full_consistent_hierarchy': full_consistent_hierarchy
677
+ }
678
+
679
+ if mode=='detailed':
680
+ for i in range(self.max_depth):
681
+ self.processed_outputs[f'level_{i+1}_labels'] = (
682
+ output_processing_class.all_level_labels()[i]
683
+ )
684
+
685
+ return self.processed_outputs
686
+
687
+ def refine_labels(self, refine_level):
688
+ """
689
+ Refine hierarchical labels to a consistent level of granularity.
690
+
691
+ This method applies post-processing rules to standardize
692
+ annotations at the specified level of granularity (broad,
693
+ medium, or fine).
694
+
695
+ Parameters
696
+ ----------
697
+ refine_level : str
698
+ Level of refinement: 'broad', 'medium', or 'fine'.
699
+
700
+ Returns
701
+ -------
702
+ list
703
+ List of refined labels at the specified level.
704
+
705
+ Raises
706
+ ------
707
+ AssertionError
708
+ If refine_level is not valid or inference hasn't been run.
709
+
710
+ Notes
711
+ -----
712
+ For 'broad' level, this returns the top-level annotations.
713
+ For 'medium' and 'fine' levels, specialized refinement is
714
+ applied.
715
+ """
716
+
717
+ assert refine_level in ['broad','medium','fine'], (
718
+ "refine_level should be 'broad', 'medium', or 'fine'."
719
+ )
720
+
721
+ assert self._inference_outputs_unprocessed is not None, (
722
+ "Labels can be refined only after inference model has been run."
723
+ )
724
+
725
+ if self._verbose:
726
+ print(
727
+ "Interpreting label predictions for consistent granularity "
728
+ f"at {refine_level} level.\n")
729
+
730
+
731
+ labels_pred = self._inference_outputs_unprocessed[
732
+ 'hierarchical_label_preds'
733
+ ]
734
+ labels_prob = self._inference_outputs_unprocessed[
735
+ 'probability_of_preds'
736
+ ]
737
+ probs = self._inference_outputs_unprocessed[
738
+ 'softmax_vals_all'
739
+ ]
740
+
741
+ # this conditional block is to specifically handle refinement
742
+ # of empty cell calls at medium and fine levels.
743
+ if refine_level in ['medium', 'fine']:
744
+ labels_pred = np.array(labels_pred, dtype=object).copy()
745
+ prev_labels = self._azimuth_refined_labels.get(
746
+ 'azimuth_broad', None
747
+ )
748
+ if prev_labels is not None:
749
+ for i, label in enumerate(labels_pred):
750
+ labels_pred[i] = [
751
+ prev_labels[i] + l[len('Empty'):]
752
+ if l.startswith('Empty') else l for l in label
753
+ ]
754
+
755
+ refine_class = PostprocessingAzimuthLabels(
756
+ labels_pred,
757
+ labels_prob,
758
+ self.max_depth,
759
+ self.num_cells,
760
+ probs,
761
+ self.inference_encoders,
762
+ refine_level,
763
+ self._model_version
764
+ )
765
+
766
+ results = refine_class.refine_labels()
767
+
768
+
769
+ (
770
+ self._azimuth_refined_labels[f'azimuth_{refine_level}']
771
+ ) = results
772
+
773
+ return results
774
+
775
+ def inference_model_embeddings(self, embedding_layer_name):
776
+ """
777
+ Extract embeddings from an intermediate layer of the inference
778
+ model.
779
+
780
+ Parameters
781
+ ----------
782
+ embedding_layer_name : str
783
+ Name of the layer to extract embeddings from.
784
+
785
+ Returns
786
+ -------
787
+ numpy.ndarray
788
+ Embeddings from the specified layer for all query cells.
789
+
790
+ Raises
791
+ ------
792
+ RuntimeError
793
+ If inference model is not found.
794
+ AssertionError
795
+ If input matrix has not been initialized.
796
+
797
+ Notes
798
+ -----
799
+ The embeddings are stored in the embeddings dictionary with a
800
+ key that combines the model name and layer name.
801
+ """
802
+
803
+ if not hasattr(self, 'inference_model'):
804
+ raise RuntimeError("inference_model not found")
805
+
806
+ assert self._inference_input_matrix is not None, (
807
+ "Input matrix not initialized. Call process_query() first."
808
+ )
809
+
810
+ embedding_class = Embeddings(self.inference_model, embedding_layer_name)
811
+
812
+ embeddings = embedding_class.embeddings(
813
+ self._inference_input_matrix,
814
+ self._eval_batch_size
815
+ )
816
+
817
+ self.embeddings[
818
+ f'{self.inference_model_name}_{embedding_layer_name}_embed'
819
+ ] = embeddings
820
+
821
+ return embeddings
822
+
823
+
824
+ def inference_model_umaps(
825
+ self,
826
+ embedding_layer_name,
827
+ n_neighbors=30,
828
+ n_components=2,
829
+ metric='cosine',
830
+ min_dist=0.3,
831
+ umap_lr=1.0,
832
+ umap_seed=42,
833
+ spread=1.0,
834
+ verbose=True,
835
+ init='spectral'
836
+ ):
837
+ """
838
+ Generate UMAP coordinates from existing embeddings.
839
+
840
+ Parameters
841
+ ----------
842
+ embedding_layer_name : str
843
+ Name of the layer whose embeddings should be used.
844
+ n_neighbors : int, default=30
845
+ Number of neighbors for UMAP.
846
+ n_components : int, default=2
847
+ Number of dimensions for UMAP output.
848
+ metric : str, default='cosine'
849
+ Distance metric for UMAP.
850
+ min_dist : float, default=0.3
851
+ Minimum distance parameter for UMAP.
852
+ umap_lr : float, default=1.0
853
+ Learning rate for UMAP.
854
+ umap_seed : int, default=42
855
+ Random seed for reproducibility.
856
+ spread : float, default=1.0
857
+ Spread parameter for UMAP.
858
+ verbose : bool, default=True
859
+ Whether to display progress during UMAP calculation.
860
+ init : str, default='spectral'
861
+ Initialization method for UMAP.
862
+
863
+ Returns
864
+ -------
865
+ numpy.ndarray
866
+ UMAP coordinates for all query cells.
867
+
868
+ Raises
869
+ ------
870
+ AssertionError
871
+ If the specified embeddings have not been generated.
872
+
873
+ Notes
874
+ -----
875
+ This method requires that embeddings have already been generated
876
+ using inference_model_embeddings().
877
+ """
878
+
879
+ embedding_key_base = (
880
+ f"{self.inference_model_name}_{embedding_layer_name}"
881
+ )
882
+ embedding_key = embedding_key_base + '_embed'
883
+ umap_key = embedding_key_base + '_umap'
884
+
885
+ assert embedding_key in self.embeddings.keys(), (
886
+ f"Embedding '{embedding_key}' not found. Generate embeddings "
887
+ "before creating umap."
888
+ )
889
+
890
+ umap_class = Umaps(
891
+ n_neighbors=n_neighbors,
892
+ n_components=n_components,
893
+ metric=metric,
894
+ min_dist=min_dist,
895
+ umap_lr=umap_lr,
896
+ umap_seed=umap_seed,
897
+ spread=spread,
898
+ verbose=verbose,
899
+ init=init
900
+ )
901
+
902
+ umap_gen = umap_class.create_umap(self.embeddings[embedding_key])
903
+ self.umaps[umap_key] = umap_gen
904
+
905
+ return umap_gen
906
+
907
+ def inference_embeddings_and_umap(
908
+ self,
909
+ embedding_layer_name,
910
+ n_neighbors=30,
911
+ n_components=2,
912
+ metric='cosine',
913
+ min_dist=0.3,
914
+ umap_lr=1.0,
915
+ umap_seed=42,
916
+ spread=1.0,
917
+ verbose=True,
918
+ init='spectral'
919
+ ):
920
+ """
921
+ Generate both embeddings and UMAP coordinates in one operation.
922
+
923
+ This is a convenience method that combines the functionality of
924
+ inference_model_embeddings() and inference_model_umaps().
925
+
926
+ Parameters
927
+ ----------
928
+ embedding_layer_name : str
929
+ Name of the layer to extract embeddings from.
930
+ n_neighbors : int, default=30
931
+ Number of neighbors for UMAP.
932
+ n_components : int, default=2
933
+ Number of dimensions for UMAP output.
934
+ metric : str, default='cosine'
935
+ Distance metric for UMAP.
936
+ min_dist : float, default=0.3
937
+ Minimum distance parameter for UMAP.
938
+ umap_lr : float, default=1.0
939
+ Learning rate for UMAP.
940
+ umap_seed : int, default=42
941
+ Random seed for reproducibility.
942
+ spread : float, default=1.0
943
+ Spread parameter for UMAP.
944
+ verbose : bool, default=True
945
+ Whether to display progress during UMAP calculation.
946
+ init : str, default='spectral'
947
+ Initialization method for UMAP.
948
+
949
+ Returns
950
+ -------
951
+ tuple
952
+ Tuple containing (embeddings, umap_coordinates).
953
+
954
+ Raises
955
+ ------
956
+ RuntimeError
957
+ If inference model is not found.
958
+ ValueError
959
+ If input matrix has not been initialized.
960
+
961
+ Notes
962
+ -----
963
+ This method may be more efficient than calling the two component
964
+ methods separately, depending on usage, as it avoids storing
965
+ intermediate results in memory twice.
966
+ """
967
+
968
+ if not hasattr(self, 'inference_model'):
969
+ raise RuntimeError("inference_model not found")
970
+
971
+ if self._inference_input_matrix is None:
972
+ raise ValueError(
973
+ "X_query has not been processed for extraction of embeddings.\n"
974
+ "Run process_query() first."
975
+ )
976
+
977
+ embeddings_and_umap_class = EmbeddingsAndUmap(
978
+ self.inference_model,
979
+ embedding_layer_name,
980
+ n_neighbors=n_neighbors,
981
+ n_components=n_components,
982
+ metric=metric,
983
+ min_dist=min_dist,
984
+ umap_lr=umap_lr,
985
+ umap_seed=umap_seed,
986
+ spread=spread,
987
+ verbose=verbose,
988
+ init=init
989
+ )
990
+
991
+ em, umap_em = embeddings_and_umap_class.create_embeddings_and_umap(
992
+ self._inference_input_matrix,
993
+ self._eval_batch_size
994
+ )
995
+
996
+ embed_key = f'{self.inference_model_name}_{embedding_layer_name}_embed'
997
+ umap_key = f'{self.inference_model_name}_{embedding_layer_name}_umap'
998
+
999
+ self.embeddings[embed_key] = em
1000
+ self.umaps[umap_key] = umap_em
1001
+
1002
+ return em, umap_em
1003
+
1004
+ def update_cells_meta(self):
1005
+ """
1006
+ Update cells_meta DataFrame with inference results.
1007
+
1008
+ This method adds or updates columns in the cell metadata DataFrame
1009
+ with inference results, including predictions and refined labels.
1010
+
1011
+ Returns
1012
+ -------
1013
+ pandas.DataFrame
1014
+ Updated cell metadata DataFrame.
1015
+
1016
+ Raises
1017
+ ------
1018
+ TypeError
1019
+ If cells_meta is not a pandas DataFrame.
1020
+ RuntimeError
1021
+ If num_cells attribute is not set.
1022
+ ValueError
1023
+ If the number of values doesn't match the number of cells.
1024
+
1025
+ Notes
1026
+ -----
1027
+ If both broad level annotations and level_zero_labels exist,
1028
+ the latter is dropped to avoid duplication.
1029
+ """
1030
+
1031
+ if not isinstance(self.cells_meta, pd.DataFrame):
1032
+ raise TypeError(
1033
+ "Query cell meta is not available as pandas dataframe"
1034
+ )
1035
+
1036
+ if not hasattr(self, 'num_cells'):
1037
+ raise RuntimeError("num_cells not found")
1038
+
1039
+ if self.processed_outputs and len(self.processed_outputs) > 0:
1040
+ for col_idx, (meta_col, values) in enumerate(
1041
+ self.processed_outputs.items()
1042
+ ):
1043
+ if len(values) != self.num_cells:
1044
+ raise ValueError(
1045
+ f"Column {meta_col} has {len(values)} values but "
1046
+ f"expected {self.num_cells}"
1047
+ )
1048
+ self.cells_meta = insert_col(
1049
+ self.cells_meta,
1050
+ col_idx,
1051
+ meta_col,
1052
+ values
1053
+ )
1054
+
1055
+ if (
1056
+ len(self._azimuth_refined_labels)
1057
+ ) > 0:
1058
+ start_idx = len(self.processed_outputs or [])
1059
+ for col_idx, (refine_col, values) in enumerate(
1060
+ self._azimuth_refined_labels.items(), start=start_idx
1061
+ ):
1062
+ if len(values) != self.num_cells:
1063
+ raise ValueError(
1064
+ f"Column {refine_col} has {len(values)} values but "
1065
+ f"expected {self.num_cells}"
1066
+ )
1067
+ self.cells_meta = insert_col(
1068
+ self.cells_meta,
1069
+ col_idx,
1070
+ refine_col,
1071
+ values
1072
+ )
1073
+
1074
+ if (
1075
+ (
1076
+ "azimuth_broad" in self.cells_meta.keys()
1077
+ ) and (
1078
+ "level_zero_labels" in self.cells_meta.keys()
1079
+ )
1080
+ ):
1081
+ self.cells_meta.drop('level_zero_labels', axis=1, inplace=True)
1082
+
1083
+
1084
+ return self.cells_meta
1085
+
1086
+
1087
+ @contextmanager
1088
+ def _scoped_slice(self, start, end):
1089
+ """
1090
+ Context manager that temporarily scopes this object to a
1091
+ slice of cells for minibatched processing.
1092
+
1093
+ Saves and restores instance state so that existing methods
1094
+ (run_inference_model, calibrate_predictions, process_outputs,
1095
+ refine_labels) operate on the slice as if it were the full
1096
+ dataset. On exit, original state is restored and intermediate
1097
+ outputs from the slice are discarded.
1098
+
1099
+ Parameters
1100
+ ----------
1101
+ start : int
1102
+ Start index (inclusive) into the inference input matrix.
1103
+ end : int
1104
+ End index (exclusive) into the inference input matrix.
1105
+
1106
+ Yields
1107
+ ------
1108
+ None
1109
+
1110
+ Notes
1111
+ -----
1112
+ This is a private method intended for use by subclasses that
1113
+ implement minibatched pipelines. It does not affect embeddings
1114
+ or umaps state.
1115
+ """
1116
+ orig_inference_input_matrix = self._inference_input_matrix
1117
+ orig_num_cells = self.num_cells
1118
+ orig_inference_outputs = self._inference_outputs_unprocessed
1119
+ orig_processed_outputs = self.processed_outputs
1120
+ orig_refined_labels = self._azimuth_refined_labels
1121
+
1122
+ self._inference_input_matrix = orig_inference_input_matrix[start:end]
1123
+ self.num_cells = end - start
1124
+ self._inference_outputs_unprocessed = None
1125
+ self.processed_outputs = None
1126
+ self._azimuth_refined_labels = {}
1127
+
1128
+ try:
1129
+ yield
1130
+ finally:
1131
+ self._inference_input_matrix = orig_inference_input_matrix
1132
+ self.num_cells = orig_num_cells
1133
+ self._inference_outputs_unprocessed = orig_inference_outputs
1134
+ self.processed_outputs = orig_processed_outputs
1135
+ self._azimuth_refined_labels = orig_refined_labels
1136
+
1137
+
1138
+ def pack_adata(self, save_path = None):
1139
+ """
1140
+ Create an AnnData object with all results and optionally save to
1141
+ disk.
1142
+
1143
+ This method packages all results (expression data, metadata,
1144
+ embeddings, and UMAP coordinates) into a unified AnnData object
1145
+ for further analysis or visualization.
1146
+
1147
+ Parameters
1148
+ ----------
1149
+ save_path : str, optional
1150
+ Path to save the AnnData object as an H5AD file.
1151
+ If None, the object is created but not saved.
1152
+
1153
+ Returns
1154
+ -------
1155
+ anndata.AnnData
1156
+ AnnData object containing all query data and results.
1157
+
1158
+ Notes
1159
+ -----
1160
+ If the specified save_path already exists, a timestamp is
1161
+ appended to the filename to prevent overwriting.
1162
+ Metadata is automatically coerced to h5ad-compatible types.
1163
+ """
1164
+
1165
+
1166
+ cells_meta_coerced = coerce_metadata_types(self.cells_meta)
1167
+ features_meta_coerced = coerce_metadata_types(self.features_meta)
1168
+
1169
+
1170
+ all_embeddings = {**self.embeddings, **self.umaps}
1171
+
1172
+ adata_obj = create_anndata(
1173
+ self.X_query,
1174
+ cells_meta_coerced,
1175
+ features_meta_coerced,
1176
+ embeddings = all_embeddings
1177
+ )
1178
+
1179
+ if save_path:
1180
+ if os.path.exists(save_path):
1181
+
1182
+ base_path, ext = os.path.splitext(save_path)
1183
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
1184
+ save_path = f"{base_path}_{timestamp}{ext}"
1185
+ print(
1186
+ f"File {save_path} already exists. Adding timestamp "
1187
+ "suffix to prevent overwrite.",
1188
+ )
1189
+ print(f"Saving to: {save_path}")
1190
+ adata_obj.write(save_path)
1191
+
1192
+ self.query = adata_obj
1193
+
1194
+ return adata_obj
1195
+
1196
+ def map_to_cell_ontology(
1197
+ self,
1198
+ src_col,
1199
+ include_cl_id=False
1200
+ ):
1201
+ """
1202
+ Map annotation labels in cells_meta to Cell Ontology terms.
1203
+
1204
+ Applies the versioned cell ontology map for the model version
1205
+ used in this run to the specified column in cells_meta, adding
1206
+ one (or two) new columns immediately after src_col. Unmapped
1207
+ labels are set to 'unmapped' and a single warning is emitted
1208
+ listing all unique labels that could not be mapped.
1209
+
1210
+ Parameters
1211
+ ----------
1212
+ src_col : str
1213
+ Name of the column in cells_meta carrying the source
1214
+ annotation labels to be mapped.
1215
+ include_cl_id : bool, default False
1216
+ If True, also adds a column named {src_col}_CL_ID
1217
+ containing the CL identifier string (e.g. 'CL:0000236'),
1218
+ or 'unmapped'.
1219
+
1220
+ Returns
1221
+ -------
1222
+ pandas.DataFrame
1223
+ Updated cells_meta with new column(s) added.
1224
+
1225
+ New columns added
1226
+ -----------------
1227
+ {src_col}_CL
1228
+ CL label string for each cell, or 'unmapped'.
1229
+ {src_col}_CL_ID (only if include_cl_id=True)
1230
+ CL identifier string, or 'unmapped'.
1231
+
1232
+ Raises
1233
+ ------
1234
+ TypeError
1235
+ If cells_meta is not a pandas DataFrame.
1236
+ ValueError
1237
+ If src_col is not present in cells_meta.
1238
+
1239
+ Warns
1240
+ -----
1241
+ UserWarning
1242
+ Emitted once, listing all unique labels that could not be
1243
+ mapped to a CL term.
1244
+
1245
+ Examples
1246
+ --------
1247
+ >>> azimuth = AzimuthNN(adata)
1248
+ >>> azimuth.map_to_cell_ontology('azimuth_fine')
1249
+ >>> azimuth.map_to_cell_ontology('azimuth_broad', include_cl_id=True)
1250
+ """
1251
+ if not isinstance(self.cells_meta, pd.DataFrame):
1252
+ raise TypeError(
1253
+ "cells_meta is not available as a pandas DataFrame."
1254
+ )
1255
+
1256
+ self.cells_meta = map_to_cell_ontology(
1257
+ self.cells_meta,
1258
+ src_col=src_col,
1259
+ model_version=self._model_version,
1260
+ include_cl_id=include_cl_id
1261
+ )
1262
+
1263
+ return self.cells_meta
1264
+
1265
+
1266
+
1267
+
1268
+ ########################################################################
1269
+ ######## Object for high level interactive usage #######################
1270
+
1271
+ _VALID_REFINE_LEVELS = ['broad', 'medium', 'fine']
1272
+
1273
+ class AzimuthNN(AzimuthNN_base):
1274
+ """
1275
+ AzimuthNN: A high-level interface for a cell annotation pipeline
1276
+ based on the Azimuth neural network.
1277
+
1278
+ This class wraps around the AzimuthNN_base class to provide a
1279
+ simplified workflow for hierarchical cell type annotation based on
1280
+ single-cell RNA-seq data, handling data loading, preprocessing,
1281
+ model inference, and visualization in a streamlined manner.
1282
+
1283
+ The pipeline automatically applies confidence calibration to improve
1284
+ the reliability of prediction confidence scores using trained
1285
+ calibration models.
1286
+
1287
+ As of v0.3.0, inference, calibration, output processing, and label
1288
+ refinement are performed in minibatches to reduce peak memory usage.
1289
+ Softmax arrays and intermediate inference outputs are released after
1290
+ each minibatch is processed.
1291
+
1292
+ For more fine-grained control over the annotation process, users
1293
+ should directly use the AzimuthNN_base class.
1294
+
1295
+ Parameters
1296
+ ----------
1297
+ query_arg : Union[str, anndata.AnnData]
1298
+ Either an AnnData object containing single-cell data or a path
1299
+ to an h5ad file.
1300
+ feature_names_col : str, optional
1301
+ Column in the anndata_object.var dataframe that contains the
1302
+ gene names to use for model input. If None, assumes var_names
1303
+ are already the correct gene identifiers.
1304
+ annotation_pipeline : str, default='supervised'
1305
+ Type of annotation pipeline to use for cell type prediction.
1306
+ model_version: str, default set to match package major version.
1307
+ Model version to use.
1308
+ eval_batch_size : int, default=8192
1309
+ Batch size to use during model inference and minibatched
1310
+ annotation pipeline.
1311
+ normalization_override : bool, default=False
1312
+ If True, skips normalization check and forces processing to
1313
+ continue.
1314
+ norm_check_batch_size : int, default=1000
1315
+ Number of cells to sample for normalization check.
1316
+ output_mode : str, default='minimal'
1317
+ Controls the verbosity of output in the cell meta dataframe.
1318
+ Options are 'minimal' or 'detailed'.
1319
+ refine : list, bool, or None, default=True
1320
+ Which refinement levels to apply during initialization.
1321
+ Accepts a list containing one or more of 'broad', 'medium',
1322
+ 'fine'. If 'medium' or 'fine' is included without 'broad',
1323
+ 'broad' is added automatically as it is a prerequisite.
1324
+ Also accepts True (equivalent to ['broad', 'medium', 'fine']),
1325
+ False or None (no refinement), or an empty list (no refinement).
1326
+ As of v0.3.0, refinement is performed by default within the
1327
+ minibatched pipeline.
1328
+
1329
+ Attributes
1330
+ ----------
1331
+ cells_meta : pandas.DataFrame or None
1332
+ Cell metadata.
1333
+ embeddings : dict
1334
+ Contains embeddings extracted from the model.
1335
+ umaps : dict
1336
+ Contains UMAP projections of the embeddings.
1337
+
1338
+ Raises
1339
+ ------
1340
+ TypeError
1341
+ If query_arg is not an AnnData object or a string path,
1342
+ if normalization_override is not a bool,
1343
+ or if norm_check_batch_size is not an integer.
1344
+ ValueError
1345
+ If output_mode is not 'minimal' or 'detailed', or if refine
1346
+ contains invalid refinement levels.
1347
+
1348
+ Examples
1349
+ --------
1350
+ >>> import anndata
1351
+ >>> adata = anndata.read_h5ad('my_data.h5ad')
1352
+ >>> azimuth = AzimuthNN(adata)
1353
+ >>> embeddings = azimuth.azimuth_embed()
1354
+ >>> umap = azimuth.azimuth_umap()
1355
+ >>> cell_metadata = azimuth.cells_meta
1356
+
1357
+ Refine only at broad and fine levels:
1358
+ >>> azimuth = AzimuthNN(adata, refine=['broad', 'fine'])
1359
+
1360
+ Skip refinement entirely:
1361
+ >>> azimuth = AzimuthNN(adata, refine=False)
1362
+ """
1363
+
1364
+ def __init__(
1365
+ self,
1366
+ query_arg,
1367
+ feature_names_col=None,
1368
+ annotation_pipeline='supervised',
1369
+ model_version=model_version_default,
1370
+ eval_batch_size=8192,
1371
+ normalization_override=False,
1372
+ norm_check_batch_size=100,
1373
+ output_mode='minimal',
1374
+ refine=True
1375
+ ):
1376
+
1377
+ """
1378
+ Initialize AzimuthNN with automatic annotation pipeline execution.
1379
+
1380
+ This constructor automatically runs the complete annotation
1381
+ workflow in minibatches:
1382
+ 1. Data loading and preprocessing
1383
+ 2. For each minibatch:
1384
+ a. Model inference
1385
+ b. Confidence calibration using trained calibration models
1386
+ c. Output processing
1387
+ d. Label refinement (per refine parameter)
1388
+ e. Release of softmax arrays and intermediate outputs
1389
+ 3. Accumulate results and update cell metadata
1390
+ """
1391
+
1392
+ if (
1393
+ not isinstance(query_arg, str) and
1394
+ not isinstance(query_arg, anndata.AnnData)
1395
+ ):
1396
+ raise TypeError(
1397
+ "query argument must either be AnnData object or "
1398
+ "path to an h5ad file."
1399
+ )
1400
+
1401
+ if feature_names_col is None:
1402
+ warnings.warn(
1403
+ "Ensure that the features metadata is indexed "
1404
+ "with gene names.",
1405
+ UserWarning
1406
+ )
1407
+
1408
+ if not isinstance(normalization_override, bool):
1409
+ raise TypeError("normalization override must be a bool")
1410
+
1411
+ if not isinstance(norm_check_batch_size, int):
1412
+ raise TypeError("norm_check_batch_size must be an integer")
1413
+
1414
+ if output_mode not in ['minimal', 'detailed']:
1415
+ raise ValueError(
1416
+ "output_mode must be either 'minimal' or 'detailed'"
1417
+ )
1418
+
1419
+ self._refine_levels = self._parse_refine_arg(refine)
1420
+
1421
+ self._query_arg = query_arg
1422
+ self._normalization_override = normalization_override
1423
+ self._norm_check_batch_size = norm_check_batch_size
1424
+ self._output_mode = output_mode
1425
+
1426
+ super().__init__(
1427
+ annotation_pipeline,
1428
+ model_version,
1429
+ eval_batch_size
1430
+ )
1431
+
1432
+ if isinstance(self._query_arg, anndata.AnnData):
1433
+ self.query_adata(
1434
+ self._query_arg,
1435
+ feature_names_col = feature_names_col
1436
+ )
1437
+ elif isinstance(self._query_arg, str):
1438
+ self.query_h5ad(
1439
+ self._query_arg,
1440
+ feature_names_col = feature_names_col
1441
+ )
1442
+
1443
+
1444
+
1445
+ self.process_query(
1446
+ normalization_override = self._normalization_override,
1447
+ norm_check_batch_size = self._norm_check_batch_size
1448
+ )
1449
+
1450
+
1451
+
1452
+ self._run_minibatched_pipeline()
1453
+
1454
+
1455
+ @staticmethod
1456
+ def _parse_refine_arg(refine):
1457
+ """
1458
+ Parse and validate the refine argument into an ordered list of
1459
+ refinement levels.
1460
+
1461
+ Parameters
1462
+ ----------
1463
+ refine : list, bool, or None
1464
+ Raw refine argument from __init__.
1465
+
1466
+ Returns
1467
+ -------
1468
+ list
1469
+ Ordered list of refinement levels to apply. May be empty
1470
+ if no refinement is requested.
1471
+
1472
+ Raises
1473
+ ------
1474
+ TypeError
1475
+ If refine is not a list, bool, or None.
1476
+ ValueError
1477
+ If refine contains invalid refinement level strings.
1478
+ """
1479
+
1480
+ if refine is None or refine is False:
1481
+ return []
1482
+
1483
+ if refine is True:
1484
+ return ['broad', 'medium', 'fine']
1485
+
1486
+ if not isinstance(refine, list):
1487
+ raise TypeError(
1488
+ "refine must be a list of refinement levels, "
1489
+ "True, False, or None."
1490
+ )
1491
+
1492
+ if len(refine) == 0:
1493
+ return []
1494
+
1495
+ invalid = [
1496
+ level for level in refine
1497
+ if level not in _VALID_REFINE_LEVELS
1498
+ ]
1499
+ if invalid:
1500
+ raise ValueError(
1501
+ f"Invalid refinement level(s): {invalid}. "
1502
+ f"Valid options are {_VALID_REFINE_LEVELS}."
1503
+ )
1504
+
1505
+ # broad is a trivial prerequisite for medium and fine
1506
+ if ('medium' in refine or 'fine' in refine) and (
1507
+ 'broad' not in refine
1508
+ ):
1509
+ refine = ['broad'] + refine
1510
+
1511
+ # enforce canonical ordering
1512
+ ordered = [
1513
+ level for level in _VALID_REFINE_LEVELS
1514
+ if level in refine
1515
+ ]
1516
+
1517
+ return ordered
1518
+
1519
+
1520
+ def _run_minibatched_pipeline(self):
1521
+ """
1522
+ Run inference, calibration, output processing, and optional
1523
+ refinement in minibatches. After each minibatch, intermediate
1524
+ softmax arrays and unprocessed outputs are released via the
1525
+ _scoped_slice context manager on AzimuthNN_base.
1526
+
1527
+ Internal print messages from run_inference_model,
1528
+ calibrate_predictions, and refine_labels are suppressed
1529
+ during minibatched processing.
1530
+ """
1531
+
1532
+ n_cells = self._inference_input_matrix.shape[0]
1533
+ n_batches = (
1534
+ (n_cells + self._eval_batch_size - 1)
1535
+ // self._eval_batch_size
1536
+ )
1537
+
1538
+ if self._refine_levels:
1539
+ refine_str = ', '.join(self._refine_levels)
1540
+ else:
1541
+ refine_str = 'none'
1542
+
1543
+ print(
1544
+ f"Splitting query data into {n_batches} "
1545
+ f"evaluation batch(es) of up to \n"
1546
+ f"{self._eval_batch_size} cells.\n"
1547
+ )
1548
+ if refine_str != 'none':
1549
+ print(
1550
+ "Interpreting label predictions for consistent granularity "
1551
+ f"at {refine_str} level(s).\n"
1552
+ )
1553
+
1554
+ # accumulators for results across minibatches
1555
+ all_processed_outputs = {}
1556
+ all_refined_labels = {}
1557
+
1558
+ # suppress internal prints during minibatched processing
1559
+ self._verbose = False
1560
+
1561
+ print("Running model:")
1562
+
1563
+ for batch_idx in range(n_batches):
1564
+ start = batch_idx * self._eval_batch_size
1565
+ end = min(
1566
+ (batch_idx + 1) * self._eval_batch_size, n_cells
1567
+ )
1568
+
1569
+ with MemoryContext():
1570
+ with self._scoped_slice(start, end):
1571
+
1572
+ _ = self.run_inference_model()
1573
+ _ = self.calibrate_predictions()
1574
+ _ = self.process_outputs(mode=self._output_mode)
1575
+
1576
+ for level in self._refine_levels:
1577
+ _ = self.refine_labels(refine_level=level)
1578
+
1579
+ # collect before scope restores state
1580
+ for key, values in (
1581
+ self.processed_outputs.items()
1582
+ ):
1583
+ if key not in all_processed_outputs:
1584
+ all_processed_outputs[key] = (
1585
+ [None] * n_cells
1586
+ )
1587
+ all_processed_outputs[key][start:end] = (
1588
+ list(values)
1589
+ )
1590
+
1591
+ for key, values in (
1592
+ self._azimuth_refined_labels.items()
1593
+ ):
1594
+ if key not in all_refined_labels:
1595
+ all_refined_labels[key] = (
1596
+ [None] * n_cells
1597
+ )
1598
+ all_refined_labels[key][start:end] = (
1599
+ list(values)
1600
+ )
1601
+
1602
+ # release minibatch softmax arrays and
1603
+ # intermediate outputs before scope restores
1604
+ self._inference_outputs_unprocessed = None
1605
+
1606
+ self._verbose = True
1607
+
1608
+ # set final accumulated results
1609
+ self.processed_outputs = all_processed_outputs
1610
+ self._azimuth_refined_labels = all_refined_labels
1611
+ self.annotations = self.processed_outputs
1612
+
1613
+ _ = self.update_cells_meta()
1614
+
1615
+
1616
+ def azimuth_refine(self, refine=None):
1617
+ """
1618
+ Refine cell type annotations at multiple granularity levels.
1619
+
1620
+ .. deprecated:: 0.3.0
1621
+ Label refinement is now performed during initialization as
1622
+ part of the minibatched pipeline. This method is retained
1623
+ for backwards compatibility and will be removed in a future
1624
+ release. Use the ``refine`` parameter in :class:`AzimuthNN`
1625
+ initialization instead.
1626
+
1627
+ If a refinement level was not included at initialization,
1628
+ re-initialization is required since softmax arrays are no
1629
+ longer held in memory after the pipeline completes.
1630
+
1631
+ Parameters
1632
+ ----------
1633
+ refine : list, optional
1634
+ List of refinement levels to check. If None, checks all
1635
+ three levels ['broad', 'medium', 'fine'].
1636
+ """
1637
+
1638
+ warnings.warn(
1639
+ "azimuth_refine() is deprecated as of v0.3.0 and will be "
1640
+ "removed in a future release. Use the 'refine' parameter "
1641
+ "in AzimuthNN initialization instead.",
1642
+ DeprecationWarning,
1643
+ stacklevel=2
1644
+ )
1645
+
1646
+ if refine is None:
1647
+ requested = _VALID_REFINE_LEVELS
1648
+ else:
1649
+ requested = refine
1650
+
1651
+ missing = [
1652
+ level for level in requested
1653
+ if level not in self._refine_levels
1654
+ ]
1655
+
1656
+ if missing:
1657
+ print(
1658
+ f"Refinement level(s) {missing} were not included "
1659
+ f"at initialization. \n"
1660
+ f"To include these levels, re-initialize AzimuthNN "
1661
+ f"with refine={requested}."
1662
+ )
1663
+ return
1664
+
1665
+ pass
1666
+
1667
+ def azimuth_embed(self):
1668
+ """
1669
+ Extract embeddings from the Azimuth model's embedding layer.
1670
+
1671
+ This method extracts cell embeddings from a pre-defined layer
1672
+ in the inference model and stores them in the embeddings
1673
+ dictionary under the key 'azimuth_embed', replacing the
1674
+ original model-specific key.
1675
+
1676
+ To extract embeddings from a different layer in the model, use
1677
+ AzimuthNN_base class for more fine grained control.
1678
+
1679
+ Returns
1680
+ -------
1681
+ numpy.ndarray
1682
+ The extracted embeddings, with shape
1683
+ (n_cells, embedding_dimension).
1684
+
1685
+ Raises
1686
+ ------
1687
+ AssertionError
1688
+ If inference model hasn't been run yet.
1689
+ """
1690
+
1691
+ azimuth_embedding_layer_name = self.model_meta[
1692
+ 'inference_model_embedding_layer'
1693
+ ]
1694
+
1695
+ azimuth_embeddings = self.inference_model_embeddings(
1696
+ embedding_layer_name = azimuth_embedding_layer_name
1697
+ )
1698
+
1699
+ self.embeddings['azimuth_embed'] = azimuth_embeddings
1700
+ del self.embeddings[
1701
+ f'{self.inference_model_name}_'
1702
+ f'{azimuth_embedding_layer_name}_embed'
1703
+ ]
1704
+
1705
+ return azimuth_embeddings
1706
+
1707
+ def azimuth_umap(
1708
+ self,
1709
+ n_neighbors=30,
1710
+ n_components=2,
1711
+ metric='cosine',
1712
+ min_dist=0.3,
1713
+ umap_lr=1.0,
1714
+ umap_seed=42,
1715
+ spread=1.0,
1716
+ verbose=True,
1717
+ init='spectral'
1718
+ ):
1719
+ """
1720
+ Generate UMAP projection from Azimuth embeddings.
1721
+
1722
+ This method creates a UMAP projection from previously extracted
1723
+ Azimuth embeddings and stores it in the umaps dictionary.
1724
+
1725
+ Parameters
1726
+ ----------
1727
+ n_neighbors : int, default=30
1728
+ Number of neighbors to consider for each point in UMAP.
1729
+ n_components : int, default=2
1730
+ Dimensionality of the UMAP projection.
1731
+ metric : str, default='cosine'
1732
+ Distance metric to use for UMAP.
1733
+ min_dist : float, default=0.3
1734
+ Minimum distance between points in the UMAP projection.
1735
+ umap_lr : float, default=1.0
1736
+ UMAP learning rate.
1737
+ umap_seed : int, default=42
1738
+ Random seed for UMAP for reproducibility.
1739
+ spread : float, default=1.0
1740
+ Scales the effective scale of embedded points.
1741
+ verbose : bool, default=True
1742
+ Whether to display progress during UMAP computation.
1743
+ init : str, default='spectral'
1744
+ Initialization method for UMAP.
1745
+
1746
+ Returns
1747
+ -------
1748
+ numpy.ndarray
1749
+ The UMAP projection, with shape (n_cells, n_components).
1750
+
1751
+ Raises
1752
+ ------
1753
+ AssertionError
1754
+ If 'azimuth_embed' embeddings haven't been generated yet.
1755
+ """
1756
+
1757
+ assert 'azimuth_embed' in self.embeddings.keys(), (
1758
+ "Extract azimuth_embed first before creating umap."
1759
+ )
1760
+
1761
+ umap_class = Umaps(
1762
+ n_neighbors=n_neighbors,
1763
+ n_components=n_components,
1764
+ metric=metric,
1765
+ min_dist=min_dist,
1766
+ umap_lr=umap_lr,
1767
+ umap_seed=umap_seed,
1768
+ spread=spread,
1769
+ verbose=verbose,
1770
+ init=init
1771
+ )
1772
+
1773
+ umap_gen = umap_class.create_umap(
1774
+ self.embeddings['azimuth_embed']
1775
+ )
1776
+ self.umaps['azimuth_umap'] = umap_gen
1777
+
1778
+ return umap_gen
1779
+
1780
+ def azimuth_embed_and_umap(
1781
+ self,
1782
+ n_neighbors=30,
1783
+ n_components=2,
1784
+ metric='cosine',
1785
+ min_dist=0.3,
1786
+ umap_lr=1.0,
1787
+ umap_seed=42,
1788
+ spread=1.0,
1789
+ verbose=True,
1790
+ init='spectral'
1791
+ ):
1792
+ """
1793
+ Extract embeddings and generate UMAP projection in one step.
1794
+
1795
+ This method provides a convenient wrapper that combines the
1796
+ functionality of azimuth_embed() and azimuth_umap() methods.
1797
+ It extracts embeddings from the inference model and immediately
1798
+ computes a UMAP projection, storing both results.
1799
+
1800
+ Parameters
1801
+ ----------
1802
+ n_neighbors : int, default=30
1803
+ Number of neighbors to consider for each point in UMAP.
1804
+ n_components : int, default=2
1805
+ Dimensionality of the UMAP projection.
1806
+ metric : str, default='cosine'
1807
+ Distance metric to use for UMAP.
1808
+ min_dist : float, default=0.3
1809
+ Minimum distance between points in the UMAP projection.
1810
+ umap_lr : float, default=1.0
1811
+ UMAP learning rate.
1812
+ umap_seed : int, default=42
1813
+ Random seed for UMAP for reproducibility.
1814
+ spread : float, default=1.0
1815
+ Scales the effective scale of embedded points.
1816
+ verbose : bool, default=True
1817
+ Whether to display progress during UMAP computation.
1818
+ init : str, default='spectral'
1819
+ Initialization method for UMAP.
1820
+
1821
+ Returns
1822
+ -------
1823
+ tuple
1824
+ A tuple containing:
1825
+ - numpy.ndarray: The extracted embeddings
1826
+ - numpy.ndarray: The UMAP projection
1827
+
1828
+ Raises
1829
+ ------
1830
+ AssertionError
1831
+ If inference model hasn't been run yet.
1832
+ """
1833
+
1834
+ azimuth_embedding_layer_name = self.model_meta[
1835
+ 'inference_model_embedding_layer'
1836
+ ]
1837
+
1838
+ (
1839
+ azimuth_embeddings,
1840
+ azimuth_umap
1841
+ ) = self.inference_embeddings_and_umap(
1842
+ embedding_layer_name = azimuth_embedding_layer_name,
1843
+ n_neighbors=n_neighbors,
1844
+ n_components=n_components,
1845
+ metric=metric,
1846
+ min_dist=min_dist,
1847
+ umap_lr=umap_lr,
1848
+ umap_seed=umap_seed,
1849
+ spread=spread,
1850
+ verbose=verbose,
1851
+ init=init
1852
+ )
1853
+
1854
+ embed_key_og = (
1855
+ f'{self.inference_model_name}_'
1856
+ f'{azimuth_embedding_layer_name}_embed'
1857
+ )
1858
+ umap_key_og = (
1859
+ f'{self.inference_model_name}_'
1860
+ f'{azimuth_embedding_layer_name}_umap'
1861
+ )
1862
+
1863
+ self.embeddings['azimuth_embed'] = azimuth_embeddings
1864
+ self.umaps['azimuth_umap'] = azimuth_umap
1865
+
1866
+ del self.embeddings[embed_key_og]
1867
+ del self.umaps[umap_key_og]
1868
+
1869
+ return azimuth_embeddings, azimuth_umap
1870
+
1871
+
1872
+
1873
+
1874
+ ################################################################################
1875
+ ########### functions for python and R script ##################################
1876
+ ################################################################################
1877
+ ################################################################################
1878
+
1879
+
1880
+
1881
+
1882
+
1883
+ ########### annotate_core, core of python/R scripts ############################
1884
+
1885
+
1886
+
1887
+ def annotate_core(
1888
+ X_query,
1889
+ query_features,
1890
+ cells_meta,
1891
+ annotation_pipeline,
1892
+ eval_batch_size,
1893
+ normalization_override,
1894
+ norm_check_batch_size,
1895
+ output_mode,
1896
+ refine_labels,
1897
+ map_to_cl,
1898
+ include_cl_id,
1899
+ extract_embeddings,
1900
+ umap_embeddings,
1901
+ n_neighbors,
1902
+ n_components,
1903
+ metric,
1904
+ min_dist,
1905
+ umap_lr,
1906
+ umap_seed,
1907
+ spread,
1908
+ verbose,
1909
+ init,
1910
+ model_version=model_version_default
1911
+ # adding a default here, so the R script does not need mods
1912
+ # to access the default.
1913
+ ):
1914
+ """
1915
+ Core function for cell type annotation using the Azimuth neural
1916
+ network, designed primarily for script-based usage.
1917
+
1918
+ While AzimuthNN and AzimuthNN_base classes provide interactive
1919
+ functionality for exploratory analysis, this function offers a
1920
+ one-step method for automated annotation via Python or R scripts. It
1921
+ performs the complete annotation workflow in a single function
1922
+ call: data preprocessing, model inference, confidence calibration,
1923
+ label generation, optional label refinement, optional Cell Ontology
1924
+ mapping, and optional embedding/UMAP generation.
1925
+
1926
+ As of v0.3.0, this function uses the AzimuthNN class internally,
1927
+ which performs inference, calibration, and refinement in minibatches
1928
+ for improved memory efficiency.
1929
+
1930
+ Parameters
1931
+ ----------
1932
+ X_query : scipy.sparse.csr_matrix
1933
+ Expression matrix with cells as rows and genes as columns.
1934
+ query_features : list of str
1935
+ List of feature names (gene identifiers) corresponding to
1936
+ columns in X_query.
1937
+ cells_meta : pandas.DataFrame
1938
+ Metadata for cells, with rows corresponding to cells in X_query.
1939
+ annotation_pipeline : str
1940
+ Type of annotation pipeline to use for cell type prediction.
1941
+ eval_batch_size : int
1942
+ Batch size to use during model inference.
1943
+ normalization_override : bool
1944
+ If True, skips normalization check and forces processing to
1945
+ continue.
1946
+ norm_check_batch_size : int
1947
+ Number of cells to sample for normalization check.
1948
+ output_mode : str
1949
+ Controls the verbosity of output in the cell meta dataframe.
1950
+ Options are 'minimal' or 'detailed'.
1951
+ refine_labels : bool
1952
+ Whether to perform label refinement at broad, medium, and fine
1953
+ levels.
1954
+ map_to_cl : list of str or None
1955
+ List of column names in cells_meta to map to Cell Ontology
1956
+ terms after annotation is complete. Each named column must
1957
+ exist in cells_meta at the time of mapping, so columns produced
1958
+ by the annotation pipeline (e.g. 'azimuth_broad', 'azimuth_fine')
1959
+ are valid targets. If None, no mapping is applied.
1960
+ include_cl_id : bool
1961
+ If True, also adds a CL identifier column (e.g. 'CL:0000236')
1962
+ alongside each CL label column produced by map_to_cl. Has no
1963
+ effect if map_to_cl is None.
1964
+ extract_embeddings : bool
1965
+ Whether to extract embeddings from the model.
1966
+ umap_embeddings : bool
1967
+ Whether to generate UMAP projections from the embeddings.
1968
+ Requires extract_embeddings=True.
1969
+ n_neighbors : int
1970
+ Number of neighbors to consider for each point in UMAP.
1971
+ n_components : int
1972
+ Dimensionality of the UMAP projection.
1973
+ metric : str
1974
+ Distance metric to use for UMAP.
1975
+ min_dist : float
1976
+ Minimum distance between points in the UMAP projection.
1977
+ umap_lr : float
1978
+ UMAP learning rate.
1979
+ umap_seed : int
1980
+ Random seed for UMAP for reproducibility.
1981
+ spread : float
1982
+ Scales the effective scale of embedded points.
1983
+ verbose : bool
1984
+ Whether to display progress during UMAP computation.
1985
+ init : str
1986
+ Initialization method for UMAP.
1987
+ model_version : str
1988
+ Model version to use, e.g. 'v0' or 'v1'. Defaults to
1989
+ model_version_default as defined in this module.
1990
+
1991
+ Returns
1992
+ -------
1993
+ dict
1994
+ A dictionary containing:
1995
+ - 'azimuth_object': The instantiated AzimuthNN object
1996
+ - 'embeddings_dict': Dictionary of computed embeddings
1997
+ - 'umap_dict': Dictionary of computed UMAP projections
1998
+ - 'cells_meta': Updated cell metadata with annotations and,
1999
+ if map_to_cl was specified, Cell Ontology columns
2000
+
2001
+ Raises
2002
+ ------
2003
+ TypeError
2004
+ If normalization_override, extract_embeddings, umap_embeddings,
2005
+ refine_labels, or include_cl_id are not boolean values, if
2006
+ norm_check_batch_size is not an integer, or if map_to_cl is
2007
+ not a list of strings or None.
2008
+ ValueError
2009
+ If output_mode is not 'minimal' or 'detailed', or if
2010
+ umap_embeddings is True but extract_embeddings is False.
2011
+
2012
+ Notes
2013
+ -----
2014
+ This function is designed to be the core engine for script-based
2015
+ automated annotation workflows. Unlike the interactive AzimuthNN and
2016
+ AzimuthNN_base classes which allow step-by-step exploration and
2017
+ visualization, this function executes the entire annotation pipeline
2018
+ in one call.
2019
+
2020
+ It's particularly useful for:
2021
+ - Batch processing of multiple datasets
2022
+ - Integration into automated analysis pipelines
2023
+ - Creating wrappers for other languages (like R)
2024
+
2025
+ Cell Ontology mapping (map_to_cl) is applied after the full
2026
+ annotation pipeline completes, so columns added by the pipeline
2027
+ (e.g. 'azimuth_broad', 'azimuth_fine') can be specified directly.
2028
+
2029
+ Examples
2030
+ --------
2031
+ >>> from scipy.sparse import csr_matrix
2032
+ >>> import pandas as pd
2033
+ >>> import numpy as np
2034
+ >>>
2035
+ >>> X = csr_matrix(np.random.rand(100, 1000))
2036
+ >>> features = [f"gene_{i}" for i in range(1000)]
2037
+ >>> meta = pd.DataFrame(index=range(100))
2038
+ >>>
2039
+ >>> # Run annotation with Cell Ontology mapping
2040
+ >>> results = annotate_core(
2041
+ ... X, features, meta,
2042
+ ... annotation_pipeline='supervised',
2043
+ ... eval_batch_size=8192,
2044
+ ... normalization_override=False,
2045
+ ... norm_check_batch_size=100,
2046
+ ... output_mode='minimal',
2047
+ ... refine_labels=True,
2048
+ ... map_to_cl=['azimuth_broad', 'azimuth_fine'],
2049
+ ... include_cl_id=True,
2050
+ ... extract_embeddings=True,
2051
+ ... umap_embeddings=True,
2052
+ ... n_neighbors=30,
2053
+ ... n_components=2,
2054
+ ... metric='cosine',
2055
+ ... min_dist=0.3,
2056
+ ... umap_lr=1.0,
2057
+ ... umap_seed=42,
2058
+ ... spread=1.0,
2059
+ ... verbose=True,
2060
+ ... init='spectral'
2061
+ ... )
2062
+ >>> annotated_meta = results['cells_meta']
2063
+ >>> embeddings = results['embeddings_dict']['azimuth_embed']
2064
+ >>> umap_coords = results['umap_dict']['azimuth_umap']
2065
+ """
2066
+
2067
+ if not isinstance(normalization_override, bool):
2068
+ raise TypeError("normalization override must be a bool")
2069
+
2070
+ if not isinstance(norm_check_batch_size, int):
2071
+ raise TypeError("norm_check_batch_size must be an integer")
2072
+
2073
+ if output_mode not in ['minimal','detailed']:
2074
+ raise ValueError(
2075
+ "mode for output processing should be either "
2076
+ "'minimal' or 'detailed'"
2077
+ )
2078
+
2079
+ if not isinstance(extract_embeddings, bool):
2080
+ raise TypeError("extract_embeddings argument should be boolean")
2081
+
2082
+ if not isinstance(umap_embeddings, bool):
2083
+ raise TypeError("umap_embeddings argument should be boolean")
2084
+
2085
+ if not isinstance(refine_labels, bool):
2086
+ raise TypeError("refine_labels argument should be boolean")
2087
+
2088
+ if map_to_cl is not None:
2089
+ if not isinstance(map_to_cl, list):
2090
+ raise TypeError(
2091
+ "map_to_cl must be a list of column name strings or None."
2092
+ )
2093
+ if not all(isinstance(col, str) for col in map_to_cl):
2094
+ raise TypeError(
2095
+ "All entries in map_to_cl must be strings."
2096
+ )
2097
+
2098
+ if not isinstance(include_cl_id, bool):
2099
+ raise TypeError("include_cl_id must be a bool.")
2100
+
2101
+ if umap_embeddings:
2102
+ if not extract_embeddings:
2103
+ raise ValueError(
2104
+ "Embeddings must be extracted to create umap.\n"
2105
+ "Set extract_embeddings to True."
2106
+ )
2107
+
2108
+ # load model meta to print model name before pipeline runs
2109
+ _version_module = importlib.import_module(
2110
+ f"panhumanpy._tools.{model_version}"
2111
+ )
2112
+ _model_meta = _version_module.model_meta
2113
+
2114
+ print("Reference model and parameters:")
2115
+ print(f" Model version: {model_version}")
2116
+ print(f" Model name: {_model_meta['inference_model_name']}")
2117
+ print(f" Evaluation batch size: {eval_batch_size}")
2118
+ print(f" Extract embeddings: {extract_embeddings}")
2119
+ print(f" Run umap: {umap_embeddings}")
2120
+ print(f" Refine labels in postprocessing: {refine_labels}")
2121
+ print(f" Map to Cell Ontology columns: {map_to_cl}")
2122
+ print(f" Include CL ID: {include_cl_id}")
2123
+
2124
+ # construct a minimal AnnData from pre-extracted components.
2125
+ # this wraps references, no data is copied.
2126
+ query_adata = anndata.AnnData(
2127
+ X = X_query,
2128
+ obs = cells_meta,
2129
+ var = pd.DataFrame(index=query_features)
2130
+ )
2131
+
2132
+ azimuth = AzimuthNN(
2133
+ query_adata,
2134
+ annotation_pipeline = annotation_pipeline,
2135
+ model_version = model_version,
2136
+ eval_batch_size = eval_batch_size,
2137
+ normalization_override = normalization_override,
2138
+ norm_check_batch_size = norm_check_batch_size,
2139
+ output_mode = output_mode,
2140
+ refine = refine_labels
2141
+ )
2142
+
2143
+ if extract_embeddings:
2144
+ if umap_embeddings:
2145
+ azimuth.azimuth_embed_and_umap(
2146
+ n_neighbors=n_neighbors,
2147
+ n_components=n_components,
2148
+ metric=metric,
2149
+ min_dist=min_dist,
2150
+ umap_lr=umap_lr,
2151
+ umap_seed=umap_seed,
2152
+ spread=spread,
2153
+ verbose=verbose,
2154
+ init=init
2155
+ )
2156
+ else:
2157
+ azimuth.azimuth_embed()
2158
+
2159
+ if map_to_cl is not None:
2160
+ for col in map_to_cl:
2161
+ azimuth.map_to_cell_ontology(
2162
+ src_col=col,
2163
+ include_cl_id=include_cl_id
2164
+ )
2165
+
2166
+ core_outputs = {
2167
+ 'azimuth_object' : azimuth,
2168
+ 'embeddings_dict' : azimuth.embeddings,
2169
+ 'umap_dict' : azimuth.umaps,
2170
+ 'cells_meta' : azimuth.cells_meta
2171
+ }
2172
+
2173
+ return core_outputs
2174
+
2175
+
2176
+
2177
+
2178
+
2179
+ ############################ arg parsing ###############################
2180
+
2181
+ def arg_parse_in():
2182
+ """
2183
+ Parse command line arguments for the Azimuth cell annotation tool.
2184
+
2185
+ Sets up argument parser with all parameters required for the annotation
2186
+ pipeline, including input file handling, model configuration, and
2187
+ visualization options.
2188
+
2189
+ Returns
2190
+ -------
2191
+ argparse.Namespace
2192
+ Parsed command line arguments
2193
+ """
2194
+
2195
+ print("Parsing arguments... \n")
2196
+ print("\n")
2197
+
2198
+
2199
+ parser = argparse.ArgumentParser()
2200
+ parser.add_argument(
2201
+ "filepath",
2202
+ help=(
2203
+ "enter abs file path to the query."
2204
+ " Query should be in h5ad format."
2205
+ ),
2206
+ type=str
2207
+ )
2208
+
2209
+
2210
+ parser.add_argument(
2211
+ "-fn",
2212
+ "--feature_names_col",
2213
+ default=None,
2214
+ help=(
2215
+ "enter the column name where the "
2216
+ "feature names are stored in query.var"
2217
+ " where query is the anndata object read "
2218
+ "from the h5ad."
2219
+ ),
2220
+ type=str
2221
+ )
2222
+
2223
+ parser.add_argument(
2224
+ "-ap",
2225
+ "--annotation_pipeline",
2226
+ default='supervised',
2227
+ help=(
2228
+ "enter annotation pipeline"
2229
+ ),
2230
+ type=str
2231
+ )
2232
+
2233
+ parser.add_argument(
2234
+ "-mv",
2235
+ "--model_version",
2236
+ default=model_version_default,
2237
+ help=(
2238
+ "enter model version to use"
2239
+ ),
2240
+ type=str
2241
+ )
2242
+
2243
+ parser.add_argument(
2244
+ "-ebs",
2245
+ "--eval_batch_size",
2246
+ default=8192,
2247
+ help=(
2248
+ "enter the evaluation batch size suitable to "
2249
+ "your hardware, defaults to 8192"
2250
+ ),
2251
+ type=int
2252
+ )
2253
+
2254
+ parser.add_argument(
2255
+ "-norm",
2256
+ "--normalization_override",
2257
+ action = "store_true",
2258
+ help="Override normalisation."
2259
+ )
2260
+
2261
+ parser.add_argument(
2262
+ "-ncbs",
2263
+ "--norm_check_batch_size",
2264
+ default=100,
2265
+ help=(
2266
+ "enter the number of cells over which "
2267
+ "normalization will be verified, defaults "
2268
+ "to 100"
2269
+ ),
2270
+ type=int
2271
+ )
2272
+
2273
+ parser.add_argument(
2274
+ "-om",
2275
+ "--output_mode",
2276
+ default='minimal',
2277
+ help=(
2278
+ "enter output mode, must be either 'minimal'"
2279
+ " or 'detailed'."
2280
+ ),
2281
+ type=str
2282
+ )
2283
+
2284
+ parser.add_argument(
2285
+ "-rf",
2286
+ "--refine_labels",
2287
+ action = "store_false",
2288
+ help="Skip label refinement."
2289
+ )
2290
+
2291
+ parser.add_argument(
2292
+ "-mcl",
2293
+ "--map_to_cl",
2294
+ default=None,
2295
+ nargs='+',
2296
+ help=(
2297
+ "One or more column names in the cell "
2298
+ "metadata to map to Cell Ontology labels. "
2299
+ "e.g. -mcl azimuth_broad azimuth_fine"
2300
+ ),
2301
+ type=str
2302
+ )
2303
+
2304
+ parser.add_argument(
2305
+ "-clid",
2306
+ "--include_cl_id",
2307
+ action="store_true",
2308
+ help=(
2309
+ "If set, also adds CL ID columns "
2310
+ "(e.g. CL:0000236) alongside CL label "
2311
+ "columns. Default: False."
2312
+ )
2313
+ )
2314
+
2315
+ parser.add_argument(
2316
+ "-em",
2317
+ "--extract_embeddings",
2318
+ action = "store_false",
2319
+ help="Skip embedding extraction"
2320
+ )
2321
+
2322
+
2323
+ parser.add_argument(
2324
+ "-umap",
2325
+ "--umap_embeddings",
2326
+ action = "store_false",
2327
+ help="Skip UMAP creation"
2328
+ )
2329
+
2330
+
2331
+
2332
+ parser.add_argument(
2333
+ "-nnbrs",
2334
+ "--n_neighbors",
2335
+ default=30,
2336
+ help=(
2337
+ "n_neighbors param for umaps, defaults "
2338
+ "to Seurat default 30"
2339
+ ),
2340
+ type=int
2341
+ )
2342
+
2343
+ parser.add_argument(
2344
+ "-nc",
2345
+ "--n_components",
2346
+ default=2,
2347
+ help=(
2348
+ "n_components param for umaps, defaults "
2349
+ "to Seurat default 2"
2350
+ ),
2351
+ type=int
2352
+ )
2353
+
2354
+ parser.add_argument(
2355
+ "-me",
2356
+ "--metric",
2357
+ default='cosine',
2358
+ help=(
2359
+ "metric param for umaps, defaults to "
2360
+ "Seurat default 'cosine'"
2361
+ ),
2362
+ type=str
2363
+ )
2364
+
2365
+ parser.add_argument(
2366
+ "-mdt",
2367
+ "--min_dist",
2368
+ default=0.3,
2369
+ help=(
2370
+ "min_dist param for umaps, defaults to "
2371
+ "Seurat default 0.3"
2372
+ ),
2373
+ type=float
2374
+ )
2375
+
2376
+ parser.add_argument(
2377
+ "-ulr",
2378
+ "--umap_lr",
2379
+ default=1.0,
2380
+ help=("learning_rate param for umaps, defaults "
2381
+ "to Seurat default 1.0"
2382
+ ),
2383
+ type=float
2384
+ )
2385
+
2386
+ parser.add_argument(
2387
+ "-useed",
2388
+ "--umap_seed",
2389
+ default=42,
2390
+ help=(
2391
+ "random_state param for reproducibility of "
2392
+ "umaps, defaults to Seurat default 42"
2393
+ ),
2394
+ type=int
2395
+ )
2396
+
2397
+ parser.add_argument(
2398
+ "-sp",
2399
+ "--spread",
2400
+ default=1.0,
2401
+ help=(
2402
+ "spread param for umaps, defaults to "
2403
+ "Seurat default 1.0"
2404
+ ),
2405
+ type=float
2406
+ )
2407
+
2408
+ parser.add_argument(
2409
+ "-uv",
2410
+ "--umap_verbose",
2411
+ action="store_false",
2412
+ help="Hide UMAP progress"
2413
+ )
2414
+
2415
+ parser.add_argument(
2416
+ "-uin",
2417
+ "--umap_init",
2418
+ default="spectral",
2419
+ help=(
2420
+ "init param for umaps, defaults to "
2421
+ "'spectral', the other option is 'random'"
2422
+ ),
2423
+ type=str
2424
+ )
2425
+
2426
+ parser.set_defaults(
2427
+ normalization_override=False,
2428
+ refine_labels=True,
2429
+ include_cl_id=False,
2430
+ extract_embeddings=True,
2431
+ umap_embeddings=True,
2432
+ umap_verbose=True
2433
+ )
2434
+
2435
+
2436
+ args = parser.parse_args()
2437
+
2438
+ return args
2439
+
2440
+
2441
+
2442
+
2443
+ def arg_parse_out(args):
2444
+ """
2445
+ Convert parsed arguments to a dictionary for the annotation pipeline.
2446
+
2447
+ Takes the parsed command line arguments and transforms them into a
2448
+ structured dictionary that can be passed to the annotation functions.
2449
+
2450
+ Parameters
2451
+ ----------
2452
+ args : argparse.Namespace
2453
+ Parsed command line arguments from arg_parse_in()
2454
+
2455
+ Returns
2456
+ -------
2457
+ dict
2458
+ Dictionary of arguments ready for use in annotation functions
2459
+ """
2460
+ print("Reading arguments... \n")
2461
+ print("\n")
2462
+
2463
+
2464
+
2465
+ query_filepath = args.filepath
2466
+ feature_names_col = args.feature_names_col
2467
+ annotation_pipeline = args.annotation_pipeline
2468
+ model_version = args.model_version
2469
+ eval_batch_size = args.eval_batch_size
2470
+ normalization_override = args.normalization_override
2471
+ norm_check_batch_size = args.norm_check_batch_size
2472
+ output_mode = args.output_mode
2473
+ refine_labels = args.refine_labels
2474
+ map_to_cl = args.map_to_cl
2475
+ include_cl_id = args.include_cl_id
2476
+ extract_embeddings = args.extract_embeddings
2477
+ umap_embeddings = args.umap_embeddings
2478
+ n_neighbors = args.n_neighbors
2479
+ n_components = args.n_components
2480
+ metric = args.metric
2481
+ min_dist = args.min_dist
2482
+ umap_lr = args.umap_lr
2483
+ umap_seed = args.umap_seed
2484
+ spread = args.spread
2485
+ umap_verbose = args.umap_verbose
2486
+ umap_init = args.umap_init
2487
+
2488
+ arguments={
2489
+ 'query_filepath' : query_filepath,
2490
+ 'feature_names_col' : feature_names_col,
2491
+ 'annotation_pipeline' : annotation_pipeline,
2492
+ 'model_version': model_version,
2493
+ 'eval_batch_size' : eval_batch_size,
2494
+ 'normalization_override' : normalization_override,
2495
+ 'norm_check_batch_size' : norm_check_batch_size,
2496
+ 'output_mode' : output_mode,
2497
+ 'refine_labels' : refine_labels,
2498
+ 'map_to_cl' : map_to_cl,
2499
+ 'include_cl_id' : include_cl_id,
2500
+ 'extract_embeddings' : extract_embeddings,
2501
+ 'umap_embeddings' : umap_embeddings,
2502
+ 'n_neighbors' : n_neighbors,
2503
+ 'n_components' : n_components,
2504
+ 'metric' : metric,
2505
+ 'min_dist' : min_dist,
2506
+ 'umap_lr' : umap_lr,
2507
+ 'umap_seed' : umap_seed,
2508
+ 'spread' : spread,
2509
+ 'verbose' : umap_verbose,
2510
+ 'init' : umap_init
2511
+ }
2512
+
2513
+ return arguments
2514
+
2515
+
2516
+ ############## annotate, executable python function ####################
2517
+
2518
+
2519
+ def annotate():
2520
+ """
2521
+ Main entry point for command-line execution of the Azimuth cell
2522
+ annotation pipeline.
2523
+
2524
+ Parses command line arguments, loads the specified h5ad file, runs
2525
+ the annotation pipeline including confidence calibration via
2526
+ annotate_core(), and saves the results as a new h5ad file in the
2527
+ same directory as the input file with '_ANN' appended to the filename.
2528
+
2529
+ This function is intended to be called when the module is executed
2530
+ directly as a script and provides a complete workflow from argument
2531
+ parsing to saving results.
2532
+
2533
+ No parameters or return values as this function is designed to be
2534
+ the executable entry point for command-line usage.
2535
+ """
2536
+
2537
+ args = arg_parse_in()
2538
+ arguments = arg_parse_out(args)
2539
+
2540
+ for key, value in arguments.items():
2541
+ globals()[key] = value
2542
+
2543
+ query_obj = ReadQueryObj(query_filepath)
2544
+ X_query = query_obj.X_query()
2545
+ query_features = query_obj.query_features(
2546
+ feature_names_col=feature_names_col
2547
+ )
2548
+ features_meta = query_obj.features_meta()
2549
+ cells_meta = query_obj.cells_meta()
2550
+
2551
+ core_outputs = annotate_core(
2552
+ X_query,
2553
+ query_features,
2554
+ cells_meta,
2555
+ annotation_pipeline,
2556
+ eval_batch_size,
2557
+ normalization_override,
2558
+ norm_check_batch_size,
2559
+ output_mode,
2560
+ refine_labels,
2561
+ map_to_cl,
2562
+ include_cl_id,
2563
+ extract_embeddings,
2564
+ umap_embeddings,
2565
+ n_neighbors,
2566
+ n_components,
2567
+ metric,
2568
+ min_dist,
2569
+ umap_lr,
2570
+ umap_seed,
2571
+ spread,
2572
+ verbose,
2573
+ init,
2574
+ model_version = model_version
2575
+ )
2576
+
2577
+ azimuth_object = core_outputs['azimuth_object']
2578
+ azimuth_object.features_meta = features_meta
2579
+
2580
+ dir_path = os.path.dirname(query_filepath)
2581
+ filename = os.path.basename(query_filepath)
2582
+ filename, ext = os.path.splitext(filename)
2583
+ out_path = os.path.join(dir_path, f'{filename}_ANN{ext}')
2584
+
2585
+ azimuth_object.pack_adata(save_path=out_path)
2586
+
2587
+
2588
+ ####################################################################
2589
+
2590
+ if __name__=="__main__":
2591
+ annotate()
2592
+
2593
+
2594
+
2595
+
2596
+
2597
+
2598
+
2599
+
2600
+
2601
+
2602
+
2603
+
2604
+
2605
+
2606
+
2607
+
2608
+
2609
+
2610
+
2611
+
2612
+
2613
+
2614
+
2615
+
2616
+
2617
+
2618
+
2619
+
2620
+
2621
+
2622
+
2623
+
2624
+
2625
+
2626
+
2627
+
2628
+
2629
+
2630
+
2631
+
2632
+
2633
+
2634
+
2635
+
2636
+
2637
+
2638
+
2639
+
2640
+
2641
+
2642
+