panhumanpy 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- panhumanpy/ANNotate.py +2642 -0
- panhumanpy/ANNotate_tools.py +3931 -0
- panhumanpy/__init__.py +27 -0
- panhumanpy/_tools/__init__.py +3 -0
- panhumanpy/_tools/v0/__init__.py +14 -0
- panhumanpy/_tools/v0/calibration/__init__.py +0 -0
- panhumanpy/_tools/v0/cell_ontology_map/__init__.py +0 -0
- panhumanpy/_tools/v0/cell_ontology_map/cell_ontology_map.csv +447 -0
- panhumanpy/_tools/v0/inference_encoders/__init__.py +0 -0
- panhumanpy/_tools/v0/inference_encoders/inference_encoders.pkl +0 -0
- panhumanpy/_tools/v0/inference_feature_panel/__init__.py +0 -0
- panhumanpy/_tools/v0/inference_feature_panel/inference_feature_panel.txt +5039 -0
- panhumanpy/_tools/v0/inference_model/__init__.py +0 -0
- panhumanpy/_tools/v0/postprocessing/__init__.py +0 -0
- panhumanpy/_tools/v0/postprocessing/panhuman_annotate_broad.csv +393 -0
- panhumanpy/_tools/v0/postprocessing/panhuman_annotate_fine.csv +398 -0
- panhumanpy/_tools/v0/postprocessing/panhuman_annotate_medium.csv +393 -0
- panhumanpy/_tools/v1/__init__.py +14 -0
- panhumanpy/_tools/v1/calibration/__init__.py +0 -0
- panhumanpy/_tools/v1/calibration/custom_objects.py +151 -0
- panhumanpy/_tools/v1/calibration/temp_scaler_L_1.keras +0 -0
- panhumanpy/_tools/v1/calibration/temp_scaler_L_2.keras +0 -0
- panhumanpy/_tools/v1/calibration/temp_scaler_L_3.keras +0 -0
- panhumanpy/_tools/v1/calibration/temp_scaler_L_4.keras +0 -0
- panhumanpy/_tools/v1/calibration/temp_scaler_L_5.keras +0 -0
- panhumanpy/_tools/v1/calibration/temp_scaler_L_6.keras +0 -0
- panhumanpy/_tools/v1/calibration/temp_scaler_L_7.keras +0 -0
- panhumanpy/_tools/v1/calibration/temp_scaler_L_8.keras +0 -0
- panhumanpy/_tools/v1/cell_ontology_map/__init__.py +0 -0
- panhumanpy/_tools/v1/cell_ontology_map/cell_ontology_map.csv +447 -0
- panhumanpy/_tools/v1/inference_encoders/__init__.py +0 -0
- panhumanpy/_tools/v1/inference_encoders/inference_encoders.pkl +0 -0
- panhumanpy/_tools/v1/inference_feature_panel/__init__.py +0 -0
- panhumanpy/_tools/v1/inference_feature_panel/inference_feature_panel.txt +5055 -0
- panhumanpy/_tools/v1/inference_model/__init__.py +0 -0
- panhumanpy/_tools/v1/postprocessing/__init__.py +0 -0
- panhumanpy/_tools/v1/postprocessing/panhuman_annotate_broad.csv +395 -0
- panhumanpy/_tools/v1/postprocessing/panhuman_annotate_fine.csv +399 -0
- panhumanpy/_tools/v1/postprocessing/panhuman_annotate_medium.csv +395 -0
- panhumanpy/loss_fn.py +40 -0
- panhumanpy-0.5.0.dist-info/METADATA +95 -0
- panhumanpy-0.5.0.dist-info/RECORD +44 -0
- panhumanpy-0.5.0.dist-info/WHEEL +4 -0
- panhumanpy-0.5.0.dist-info/entry_points.txt +2 -0
panhumanpy/ANNotate.py
ADDED
|
@@ -0,0 +1,2642 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Azimuth Cell Annotation: Neural network-based hierarchical cell type
|
|
3
|
+
annotation for single-cell RNA-seq data.
|
|
4
|
+
|
|
5
|
+
This module provides tools for hierarchical cell type annotation and
|
|
6
|
+
interpretation based on single-cell RNA-seq data using the Azimuth
|
|
7
|
+
neural network model trained on annotated panhuman scRNA-seq data.
|
|
8
|
+
|
|
9
|
+
Key Components
|
|
10
|
+
-------------
|
|
11
|
+
AzimuthNN_base : Class
|
|
12
|
+
Low-level class providing fine-grained control over the annotation
|
|
13
|
+
process. Suitable for advanced users who need detailed control or
|
|
14
|
+
are processing data in batches to optimize memory usage.
|
|
15
|
+
|
|
16
|
+
AzimuthNN : Class
|
|
17
|
+
High-level interface that wraps around AzimuthNN_base for interactive
|
|
18
|
+
analysis. Provides a streamlined workflow for cell annotation with
|
|
19
|
+
sensible defaults. Recommended for most interactive analysis sessions
|
|
20
|
+
and notebooks.
|
|
21
|
+
|
|
22
|
+
annotate_core : Function
|
|
23
|
+
Core function for script-based automated annotation. Designed for
|
|
24
|
+
batch processing and integration into analysis pipelines.
|
|
25
|
+
|
|
26
|
+
Usage Examples
|
|
27
|
+
-------------
|
|
28
|
+
Interactive usage with high-level interface:
|
|
29
|
+
>>> import anndata
|
|
30
|
+
>>> from panhumanpy import AzimuthNN
|
|
31
|
+
>>> adata = anndata.read_h5ad('my_data.h5ad')
|
|
32
|
+
>>> # Run minimal annotation pipeline with calibration
|
|
33
|
+
>>> azimuth = AzimuthNN(adata)
|
|
34
|
+
>>> embeddings = azimuth.azimuth_embed() # Extract embeddings
|
|
35
|
+
>>> umap = azimuth.azimuth_umap() # Generate UMAP
|
|
36
|
+
>>> # Optionally map annotations to Cell Ontology terms
|
|
37
|
+
>>> azimuth.map_to_cell_ontology('azimuth_fine')
|
|
38
|
+
>>> azimuth.map_to_cell_ontology('azimuth_broad', include_cl_id=True)
|
|
39
|
+
>>> adata_annotated = azimuth.pack_adata('output.h5ad') # Save results
|
|
40
|
+
|
|
41
|
+
For more detailed documentation on specific classes and functions:
|
|
42
|
+
>>> help(AzimuthNN)
|
|
43
|
+
>>> help(AzimuthNN_base)
|
|
44
|
+
>>> help(annotate_core)
|
|
45
|
+
|
|
46
|
+
Command-line Usage
|
|
47
|
+
-----------------
|
|
48
|
+
This module can be run as a standalone script to annotate h5ad files:
|
|
49
|
+
|
|
50
|
+
annotate /path/to/input.h5ad [options]
|
|
51
|
+
|
|
52
|
+
Required positional argument:
|
|
53
|
+
filepath Path to input h5ad file containing
|
|
54
|
+
single-cell data
|
|
55
|
+
|
|
56
|
+
Optional arguments:
|
|
57
|
+
-fn, --feature_names_col
|
|
58
|
+
Column in query.var containing gene
|
|
59
|
+
names (default: None)
|
|
60
|
+
-ap, --annotation_pipeline
|
|
61
|
+
Annotation pipeline to use
|
|
62
|
+
(default: 'supervised')
|
|
63
|
+
-ebs, --eval_batch_size
|
|
64
|
+
Batch size for model inference
|
|
65
|
+
(default: 8192)
|
|
66
|
+
-norm, --normalization_override
|
|
67
|
+
Skip normalization check (default: False)
|
|
68
|
+
-ncbs, --norm_check_batch_size
|
|
69
|
+
Number of cells to sample for normalization
|
|
70
|
+
check (default: 100)
|
|
71
|
+
-om, --output_mode Output verbosity, 'minimal' or 'detailed'
|
|
72
|
+
(default: 'minimal')
|
|
73
|
+
-rf, --refine_labels Skip hierarchical label refinement
|
|
74
|
+
(default: use refinement)
|
|
75
|
+
-mcl, --map_to_cl One or more column names in the cell metadata
|
|
76
|
+
to map to Cell Ontology labels after annotation.
|
|
77
|
+
Multiple columns can be provided as a
|
|
78
|
+
space-separated list.
|
|
79
|
+
e.g. -mcl azimuth_broad azimuth_fine
|
|
80
|
+
(default: None, no mapping applied)
|
|
81
|
+
-clid, --include_cl_id
|
|
82
|
+
If set, also adds CL ID columns
|
|
83
|
+
(e.g. CL:0000236) alongside CL label columns
|
|
84
|
+
produced by --map_to_cl. Has no effect if
|
|
85
|
+
--map_to_cl is not specified.
|
|
86
|
+
(default: False)
|
|
87
|
+
-em, --extract_embeddings
|
|
88
|
+
Skip neural network embeddings extraction
|
|
89
|
+
(default: extract embeddings)
|
|
90
|
+
-umap, --umap_embeddings
|
|
91
|
+
Skip UMAP projection generation
|
|
92
|
+
(default: generate UMAP)
|
|
93
|
+
|
|
94
|
+
UMAP parameters:
|
|
95
|
+
-nnbrs, --n_neighbors Neighbors per point in UMAP (default: 30)
|
|
96
|
+
-nc, --n_components UMAP dimensionality (default: 2)
|
|
97
|
+
-me, --metric Distance metric for UMAP (default: 'cosine')
|
|
98
|
+
-mdt, --min_dist Minimum distance in UMAP (default: 0.3)
|
|
99
|
+
-ulr, --umap_lr UMAP learning rate (default: 1.0)
|
|
100
|
+
-useed, --umap_seed Random seed for reproducibility (default: 42)
|
|
101
|
+
-sp, --spread UMAP spread parameter (default: 1.0)
|
|
102
|
+
-uv, --umap_verbose Hide UMAP progress
|
|
103
|
+
(default: show progress)
|
|
104
|
+
-uin, --umap_init UMAP initialization method (default: 'spectral')
|
|
105
|
+
|
|
106
|
+
Output:
|
|
107
|
+
The annotated data will be saved as a new h5ad file in the same directory
|
|
108
|
+
as the input file, with '_ANN' appended to the filename. If a file with
|
|
109
|
+
that name already exists, a timestamp (YYYYMMDD_HHMMSS) will be
|
|
110
|
+
automatically appended to prevent overwriting existing results.
|
|
111
|
+
|
|
112
|
+
Example commands:
|
|
113
|
+
annotate my_cells.h5ad -fn feature_name -ebs 4096 -nc 3
|
|
114
|
+
annotate my_cells.h5ad -mcl azimuth_broad azimuth_fine -clid
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
from panhumanpy.ANNotate_tools import *
|
|
120
|
+
|
|
121
|
+
model_version_default = 'v0'
|
|
122
|
+
|
|
123
|
+
_gpu_configured = False
|
|
124
|
+
|
|
125
|
+
def configure_once():
|
|
126
|
+
"""
|
|
127
|
+
Configures TensorFlow GPU settings once per process.
|
|
128
|
+
This wrapper ensures the configuration only happens once.
|
|
129
|
+
"""
|
|
130
|
+
global _gpu_configured
|
|
131
|
+
if not _gpu_configured:
|
|
132
|
+
configure()
|
|
133
|
+
_gpu_configured = True
|
|
134
|
+
return True
|
|
135
|
+
return False
|
|
136
|
+
|
|
137
|
+
configure_once()
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
########################################################################
|
|
141
|
+
###### Base class for low level interactive usage ######################
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class AzimuthNN_base(AutoloadInferenceTools):
|
|
145
|
+
"""
|
|
146
|
+
Base class for low-level interactive usage of the Azimuth neural
|
|
147
|
+
network annotation pipeline.
|
|
148
|
+
|
|
149
|
+
This class provides a comprehensive framework for single-cell
|
|
150
|
+
RNA-seq annotation using neural network models. It handles the
|
|
151
|
+
complete workflow from data loading and preprocessing to inference,
|
|
152
|
+
confidence calibration, post-processing, and result visualization.
|
|
153
|
+
|
|
154
|
+
This includes functionality for extracting embeddings, generating
|
|
155
|
+
UMAP visualizations, and refining annotations at different levels of
|
|
156
|
+
granularity.
|
|
157
|
+
|
|
158
|
+
Parameters
|
|
159
|
+
----------
|
|
160
|
+
annotation_pipeline : str, default='supervised'
|
|
161
|
+
The type of annotation pipeline to use.
|
|
162
|
+
model_version: str, default set to match package major version.
|
|
163
|
+
Model version to use.
|
|
164
|
+
eval_batch_size : int, default=8192
|
|
165
|
+
Batch size for inference and embedding generation.
|
|
166
|
+
|
|
167
|
+
Attributes
|
|
168
|
+
----------
|
|
169
|
+
query : anndata.AnnData or None
|
|
170
|
+
The AnnData object if loaded.
|
|
171
|
+
X_query : scipy.sparse.csr_matrix or None
|
|
172
|
+
Expression matrix in CSR format.
|
|
173
|
+
query_features : list or None
|
|
174
|
+
List of feature names.
|
|
175
|
+
features_meta : pandas.DataFrame or None
|
|
176
|
+
Feature metadata.
|
|
177
|
+
cells_meta : pandas.DataFrame or None
|
|
178
|
+
Cell metadata.
|
|
179
|
+
num_cells : int or None
|
|
180
|
+
Number of cells in the query.
|
|
181
|
+
processed_outputs : dict or None
|
|
182
|
+
Processed inference results.
|
|
183
|
+
embeddings : dict
|
|
184
|
+
Dictionary of extracted embeddings.
|
|
185
|
+
umaps : dict
|
|
186
|
+
Dictionary of generated UMAP coordinates.
|
|
187
|
+
|
|
188
|
+
Raises
|
|
189
|
+
------
|
|
190
|
+
TypeError
|
|
191
|
+
If input parameters are not of the correct type.
|
|
192
|
+
RuntimeError
|
|
193
|
+
If model metadata fails to load.
|
|
194
|
+
|
|
195
|
+
Notes
|
|
196
|
+
-----
|
|
197
|
+
This class is designed for programmatic use and provides
|
|
198
|
+
fine-grained control over each step of the annotation pipeline.
|
|
199
|
+
Consider using a higher-level interface for convenience if a
|
|
200
|
+
standard workflow is sufficient for your needs.
|
|
201
|
+
"""
|
|
202
|
+
def __init__(
|
|
203
|
+
self,
|
|
204
|
+
annotation_pipeline='supervised',
|
|
205
|
+
model_version=model_version_default,
|
|
206
|
+
eval_batch_size=8192,
|
|
207
|
+
):
|
|
208
|
+
|
|
209
|
+
if not isinstance(annotation_pipeline, str):
|
|
210
|
+
raise TypeError("annotation_pipeline must be a string")
|
|
211
|
+
|
|
212
|
+
if not isinstance(model_version, str):
|
|
213
|
+
raise TypeError("model_version must be a string")
|
|
214
|
+
|
|
215
|
+
if not isinstance(eval_batch_size, int):
|
|
216
|
+
raise TypeError("eval_batch_size must be an integer")
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
self._annotation_pipeline = annotation_pipeline
|
|
221
|
+
self._model_version = model_version
|
|
222
|
+
self._eval_batch_size = eval_batch_size
|
|
223
|
+
self._verbose = True
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
super().__init__(annotation_pipeline, model_version)
|
|
227
|
+
|
|
228
|
+
if not hasattr(self, 'model_meta'):
|
|
229
|
+
raise RuntimeError("Failed to load model metadata")
|
|
230
|
+
|
|
231
|
+
for meta_key in self.model_meta.keys():
|
|
232
|
+
if not isinstance(meta_key, str):
|
|
233
|
+
raise TypeError("All model metadata keys must be strings")
|
|
234
|
+
setattr(self, meta_key, self.model_meta[meta_key])
|
|
235
|
+
|
|
236
|
+
self.query = None
|
|
237
|
+
self.X_query = None
|
|
238
|
+
self.query_features = None
|
|
239
|
+
self.features_meta = None
|
|
240
|
+
self.cells_meta = None
|
|
241
|
+
self.num_cells = None
|
|
242
|
+
self._inference_input_matrix = None
|
|
243
|
+
|
|
244
|
+
self._inference_outputs_unprocessed = None
|
|
245
|
+
self.processed_outputs = None
|
|
246
|
+
self._azimuth_refined_labels = {}
|
|
247
|
+
self.embeddings = {}
|
|
248
|
+
self.umaps = {}
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def query_stripped(
|
|
253
|
+
self,
|
|
254
|
+
X_query,
|
|
255
|
+
query_features,
|
|
256
|
+
cells_meta
|
|
257
|
+
):
|
|
258
|
+
"""
|
|
259
|
+
Load query data directly from expression matrix and metadata.
|
|
260
|
+
|
|
261
|
+
This method allows for direct loading of pre-processed
|
|
262
|
+
expression data without requiring an AnnData object. This is
|
|
263
|
+
useful for integration with custom preprocessing pipelines.
|
|
264
|
+
|
|
265
|
+
Parameters
|
|
266
|
+
----------
|
|
267
|
+
X_query : scipy.sparse.csr_matrix
|
|
268
|
+
Expression matrix with cells as rows and features as columns.
|
|
269
|
+
query_features : list of str
|
|
270
|
+
List of feature names corresponding to columns in X_query.
|
|
271
|
+
cells_meta : pandas.DataFrame
|
|
272
|
+
Cell metadata with rows corresponding to cells in X_query.
|
|
273
|
+
|
|
274
|
+
Raises
|
|
275
|
+
------
|
|
276
|
+
TypeError
|
|
277
|
+
If inputs are not of correct type.
|
|
278
|
+
ValueError
|
|
279
|
+
If dimensions of inputs don't match.
|
|
280
|
+
|
|
281
|
+
Notes
|
|
282
|
+
-----
|
|
283
|
+
This method creates a minimal features_meta DataFrame based on
|
|
284
|
+
the provided feature names.
|
|
285
|
+
"""
|
|
286
|
+
if not isinstance(X_query, csr_matrix):
|
|
287
|
+
raise TypeError("X_query must be a scipy.sparse.csr_matrix")
|
|
288
|
+
|
|
289
|
+
if not isinstance(query_features, list) or not all(isinstance(f, str)
|
|
290
|
+
for f in query_features):
|
|
291
|
+
raise TypeError("query_features must be a list of strings")
|
|
292
|
+
|
|
293
|
+
if not isinstance(cells_meta, pd.DataFrame):
|
|
294
|
+
raise TypeError("cells_meta must be a pandas DataFrame")
|
|
295
|
+
|
|
296
|
+
if len(query_features) != X_query.shape[1]:
|
|
297
|
+
raise ValueError(
|
|
298
|
+
f"Number of features ({len(query_features)}) "
|
|
299
|
+
f"does not match X_query columns ({X_query.shape[1]})"
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
if cells_meta.shape[0] != X_query.shape[0]:
|
|
303
|
+
raise ValueError(
|
|
304
|
+
f"Number of cells in metadata "
|
|
305
|
+
f"({cells_meta.shape[0]}) does not match X_query rows "
|
|
306
|
+
f"({X_query.shape[0]})"
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
if X_query.dtype != np.float64:
|
|
310
|
+
X_query = X_query.astype(np.float64)
|
|
311
|
+
|
|
312
|
+
self.X_query = X_query
|
|
313
|
+
self.query_features = query_features
|
|
314
|
+
self.features_meta = pd.DataFrame(
|
|
315
|
+
{'feature_name':query_features},
|
|
316
|
+
index = query_features
|
|
317
|
+
)
|
|
318
|
+
self.cells_meta = cells_meta
|
|
319
|
+
self.num_cells = X_query.shape[0]
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def query_adata(
|
|
323
|
+
self,
|
|
324
|
+
query_arg,
|
|
325
|
+
feature_names_col=None
|
|
326
|
+
):
|
|
327
|
+
"""
|
|
328
|
+
Load query data from an AnnData object.
|
|
329
|
+
|
|
330
|
+
Parameters
|
|
331
|
+
----------
|
|
332
|
+
query_arg : anndata.AnnData
|
|
333
|
+
AnnData object containing expression data and metadata.
|
|
334
|
+
feature_names_col : str, optional
|
|
335
|
+
Column in var DataFrame to use for feature names.
|
|
336
|
+
If None, uses the var_names index.
|
|
337
|
+
|
|
338
|
+
Notes
|
|
339
|
+
-----
|
|
340
|
+
This method extracts the expression matrix, feature names,
|
|
341
|
+
and metadata from the provided AnnData object.
|
|
342
|
+
"""
|
|
343
|
+
query_obj = QueryObj(query_arg)
|
|
344
|
+
|
|
345
|
+
self.X_query = query_obj.X_query()
|
|
346
|
+
if self.X_query.dtype != np.float64:
|
|
347
|
+
self.X_query = self.X_query.astype(np.float64)
|
|
348
|
+
|
|
349
|
+
self.query_features = query_obj.query_features(
|
|
350
|
+
feature_names_col=feature_names_col
|
|
351
|
+
)
|
|
352
|
+
self.features_meta = query_obj.features_meta()
|
|
353
|
+
self.cells_meta = query_obj.cells_meta()
|
|
354
|
+
self.num_cells = self.X_query.shape[0]
|
|
355
|
+
|
|
356
|
+
def query_h5ad(
|
|
357
|
+
self,
|
|
358
|
+
query_filepath,
|
|
359
|
+
feature_names_col=None
|
|
360
|
+
):
|
|
361
|
+
"""
|
|
362
|
+
Load query data from an H5AD file on disk.
|
|
363
|
+
|
|
364
|
+
Parameters
|
|
365
|
+
----------
|
|
366
|
+
query_filepath : str
|
|
367
|
+
Path to the H5AD file containing the query data.
|
|
368
|
+
feature_names_col : str, optional
|
|
369
|
+
Column in var DataFrame to use for feature names.
|
|
370
|
+
If None, uses the var_names index.
|
|
371
|
+
|
|
372
|
+
Raises
|
|
373
|
+
------
|
|
374
|
+
ValueError
|
|
375
|
+
If the file is not in H5AD format.
|
|
376
|
+
|
|
377
|
+
Notes
|
|
378
|
+
-----
|
|
379
|
+
This method reads the H5AD file from disk and extracts the
|
|
380
|
+
necessary components for inference.
|
|
381
|
+
"""
|
|
382
|
+
query_obj = ReadQueryObj(query_filepath)
|
|
383
|
+
|
|
384
|
+
self.X_query = query_obj.X_query()
|
|
385
|
+
if self.X_query.dtype != np.float64:
|
|
386
|
+
self.X_query = self.X_query.astype(np.float64)
|
|
387
|
+
|
|
388
|
+
self.query_features = query_obj.query_features(
|
|
389
|
+
feature_names_col=feature_names_col
|
|
390
|
+
)
|
|
391
|
+
self.features_meta = query_obj.features_meta()
|
|
392
|
+
self.cells_meta = query_obj.cells_meta()
|
|
393
|
+
self.num_cells = self.X_query.shape[0]
|
|
394
|
+
|
|
395
|
+
def process_query(
|
|
396
|
+
self,
|
|
397
|
+
normalization_override=False,
|
|
398
|
+
norm_check_batch_size=100
|
|
399
|
+
):
|
|
400
|
+
"""
|
|
401
|
+
Process the query data to prepare it for inference.
|
|
402
|
+
|
|
403
|
+
This method prepares the expression data for the inference model
|
|
404
|
+
according to the specified annotation pipeline. The processing steps
|
|
405
|
+
vary depending on the pipeline type, potentially including
|
|
406
|
+
normalization, feature selection, dimensionality reduction, or
|
|
407
|
+
other transformations.
|
|
408
|
+
|
|
409
|
+
Parameters
|
|
410
|
+
----------
|
|
411
|
+
normalization_override : bool, default=False
|
|
412
|
+
If True, bypasses normalization entirely regardless of
|
|
413
|
+
whether the values are integers or not.
|
|
414
|
+
norm_check_batch_size : int, default=100
|
|
415
|
+
Batch size for checking normalization status.
|
|
416
|
+
|
|
417
|
+
Raises
|
|
418
|
+
------
|
|
419
|
+
TypeError
|
|
420
|
+
If parameters are not of the correct type.
|
|
421
|
+
|
|
422
|
+
Notes
|
|
423
|
+
-----
|
|
424
|
+
This method must be called after loading query data and before
|
|
425
|
+
running inference or extracting embeddings. The specific processing
|
|
426
|
+
steps depend on the annotation_pipeline specified during
|
|
427
|
+
initialization.
|
|
428
|
+
|
|
429
|
+
Currently, only the 'supervised' annotation pipeline is implemented,
|
|
430
|
+
which normalizes the expression data and aligns it with a reference
|
|
431
|
+
feature panel.
|
|
432
|
+
"""
|
|
433
|
+
|
|
434
|
+
if not isinstance(normalization_override, bool):
|
|
435
|
+
raise TypeError("normalization override must be a bool")
|
|
436
|
+
|
|
437
|
+
if not isinstance(norm_check_batch_size, int):
|
|
438
|
+
raise TypeError("norm_check_batch_size must be an integer")
|
|
439
|
+
|
|
440
|
+
query_processing_class = InferenceInputData(
|
|
441
|
+
self.X_query,
|
|
442
|
+
self.query_features,
|
|
443
|
+
self.inference_feature_panel,
|
|
444
|
+
normalization_override = normalization_override,
|
|
445
|
+
norm_check_batch_size = norm_check_batch_size
|
|
446
|
+
)
|
|
447
|
+
|
|
448
|
+
self._inference_input_matrix = query_processing_class.inference_input(
|
|
449
|
+
annotation_pipeline = self._annotation_pipeline
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def run_inference_model(self):
|
|
454
|
+
"""
|
|
455
|
+
Run the inference model on the processed query data.
|
|
456
|
+
|
|
457
|
+
This method executes the neural network inference to generate
|
|
458
|
+
cell type predictions.
|
|
459
|
+
|
|
460
|
+
Returns
|
|
461
|
+
-------
|
|
462
|
+
dict
|
|
463
|
+
Dictionary of raw inference outputs including hierarchical
|
|
464
|
+
label predictions and probabilities.
|
|
465
|
+
|
|
466
|
+
Raises
|
|
467
|
+
------
|
|
468
|
+
AssertionError
|
|
469
|
+
If input matrix has not been initialized by calling
|
|
470
|
+
process_query().
|
|
471
|
+
|
|
472
|
+
Notes
|
|
473
|
+
-----
|
|
474
|
+
The raw outputs should be calibrated using calibrate_predictions()
|
|
475
|
+
and then processed using process_outputs() before further use
|
|
476
|
+
downstream. The typical workflow is:
|
|
477
|
+
1. run_inference_model()
|
|
478
|
+
2. calibrate_predictions()
|
|
479
|
+
3. process_outputs()
|
|
480
|
+
"""
|
|
481
|
+
|
|
482
|
+
assert self._inference_input_matrix is not None, (
|
|
483
|
+
"Input matrix not initialized. Call process_query() first."
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
inference_class = Inference(
|
|
487
|
+
self._inference_input_matrix,
|
|
488
|
+
self.inference_model,
|
|
489
|
+
self.inference_encoders,
|
|
490
|
+
self._eval_batch_size,
|
|
491
|
+
self.max_depth,
|
|
492
|
+
verbose = self._verbose
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
self._inference_outputs_unprocessed = inference_class.run_inference()
|
|
496
|
+
|
|
497
|
+
return self._inference_outputs_unprocessed
|
|
498
|
+
|
|
499
|
+
def calibrate_predictions(self):
|
|
500
|
+
"""
|
|
501
|
+
Apply calibration to softmax outputs using trained calibrators.
|
|
502
|
+
|
|
503
|
+
This method calibrates the softmax probability outputs from each
|
|
504
|
+
hierarchical level using the corresponding trained calibration models,
|
|
505
|
+
if available. Calibration improves the reliability and trustworthiness
|
|
506
|
+
of prediction confidence scores by adjusting for overconfidence or
|
|
507
|
+
underconfidence in the original model outputs.
|
|
508
|
+
|
|
509
|
+
Returns
|
|
510
|
+
-------
|
|
511
|
+
dict
|
|
512
|
+
Updated inference outputs dictionary with calibrated results.
|
|
513
|
+
Contains the same keys as the original inference outputs but with
|
|
514
|
+
calibrated values:
|
|
515
|
+
|
|
516
|
+
- 'softmax_vals_all': List of calibrated softmax probability arrays,
|
|
517
|
+
one per hierarchical level
|
|
518
|
+
- 'probability_of_preds': Updated maximum probability values from
|
|
519
|
+
the calibrated softmax distributions
|
|
520
|
+
- Other keys remain unchanged from the original inference outputs
|
|
521
|
+
|
|
522
|
+
Notes
|
|
523
|
+
-----
|
|
524
|
+
- Only applies calibration if calibration method is specified in model
|
|
525
|
+
metadata and calibrator models are available
|
|
526
|
+
- If no calibration is configured (calibration method is None), the
|
|
527
|
+
method returns the original inference outputs unchanged
|
|
528
|
+
- Each hierarchical level is calibrated independently using its own
|
|
529
|
+
trained calibrator model
|
|
530
|
+
- Memory management is applied during processing to handle large datasets
|
|
531
|
+
efficiently by cleaning up intermediate results after each level
|
|
532
|
+
|
|
533
|
+
Raises
|
|
534
|
+
------
|
|
535
|
+
AssertionError
|
|
536
|
+
If calibration method is not None but the number of available
|
|
537
|
+
calibrator models doesn't match the expected number of hierarchical
|
|
538
|
+
levels (max_depth).
|
|
539
|
+
|
|
540
|
+
Examples
|
|
541
|
+
--------
|
|
542
|
+
The method is typically called as part of the inference pipeline:
|
|
543
|
+
|
|
544
|
+
>>> azimuth = AzimuthNN_base()
|
|
545
|
+
>>> # ... load data and run inference ...
|
|
546
|
+
>>> raw_outputs = azimuth.run_inference_model()
|
|
547
|
+
>>> calibrated_outputs = azimuth.calibrate_predictions()
|
|
548
|
+
>>> # Calibrated outputs now have adjusted confidence scores
|
|
549
|
+
|
|
550
|
+
The calibration process transforms prediction confidence scores:
|
|
551
|
+
|
|
552
|
+
- Before calibration: Model might be over(/under)-confident
|
|
553
|
+
(high probabilities for uncertain predictions or vice-versa)
|
|
554
|
+
- After calibration: Probabilities better reflect true prediction
|
|
555
|
+
confidence and uncertainty
|
|
556
|
+
|
|
557
|
+
"""
|
|
558
|
+
|
|
559
|
+
calibration_method = self.model_meta['calibration']
|
|
560
|
+
if calibration_method is not None:
|
|
561
|
+
assert len(self.calibrators)==self.max_depth, (
|
|
562
|
+
"Calibration method is not None, expected number of "
|
|
563
|
+
f"calibrator models: {self.max_depth}, number of "
|
|
564
|
+
f"calibrator models found: {len(self.calibrators)}"
|
|
565
|
+
)
|
|
566
|
+
|
|
567
|
+
softmax_all = self._inference_outputs_unprocessed[
|
|
568
|
+
'softmax_vals_all'
|
|
569
|
+
]
|
|
570
|
+
|
|
571
|
+
calibrated_levels_cache = []
|
|
572
|
+
max_probs_levels_cache = []
|
|
573
|
+
|
|
574
|
+
for level in range(self.max_depth):
|
|
575
|
+
with MemoryContext():
|
|
576
|
+
sm_array = softmax_all[level]
|
|
577
|
+
calibrator_model = self.calibrators[level]
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
calibration_obj = CalibrationSingleClassifier(
|
|
581
|
+
softmax = sm_array,
|
|
582
|
+
eval_batch_size = self._eval_batch_size,
|
|
583
|
+
verbose = self._verbose
|
|
584
|
+
)
|
|
585
|
+
|
|
586
|
+
calibrated_sm = calibration_obj.calibrate(
|
|
587
|
+
calibration_method,
|
|
588
|
+
calibrator_model
|
|
589
|
+
)
|
|
590
|
+
|
|
591
|
+
calibrated_levels_cache.append(calibrated_sm)
|
|
592
|
+
max_probs_levels_cache.append(
|
|
593
|
+
np.max(calibrated_sm, axis=-1)
|
|
594
|
+
)
|
|
595
|
+
|
|
596
|
+
max_probs = np.column_stack(max_probs_levels_cache)
|
|
597
|
+
|
|
598
|
+
self._inference_outputs_unprocessed[
|
|
599
|
+
'softmax_vals_all'
|
|
600
|
+
] = calibrated_levels_cache
|
|
601
|
+
|
|
602
|
+
self._inference_outputs_unprocessed[
|
|
603
|
+
'probability_of_preds'
|
|
604
|
+
] = max_probs
|
|
605
|
+
|
|
606
|
+
return self._inference_outputs_unprocessed
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def process_outputs(self, mode='minimal'):
|
|
611
|
+
"""
|
|
612
|
+
Process raw inference outputs into usable predictions.
|
|
613
|
+
|
|
614
|
+
This method organizes the raw inference outputs into a structured
|
|
615
|
+
dictionary of predictions at various hierarchical levels.
|
|
616
|
+
|
|
617
|
+
Parameters
|
|
618
|
+
----------
|
|
619
|
+
mode : str, default='minimal'
|
|
620
|
+
Processing mode: 'minimal' provides essential outputs,
|
|
621
|
+
'detailed' includes additional information for all levels.
|
|
622
|
+
|
|
623
|
+
Returns
|
|
624
|
+
-------
|
|
625
|
+
dict
|
|
626
|
+
Dictionary of processed outputs including hierarchical labels,
|
|
627
|
+
level-specific labels, and confidence scores.
|
|
628
|
+
|
|
629
|
+
Raises
|
|
630
|
+
------
|
|
631
|
+
AssertionError
|
|
632
|
+
If mode is not 'minimal' or 'detailed'.
|
|
633
|
+
|
|
634
|
+
Notes
|
|
635
|
+
-----
|
|
636
|
+
This method should be called after run_inference_model() and
|
|
637
|
+
calibrate_predictions(). The calibration step improves confidence
|
|
638
|
+
score reliability by correcting for model overconfidence or
|
|
639
|
+
underconfidence using trained calibration models.
|
|
640
|
+
"""
|
|
641
|
+
|
|
642
|
+
assert mode in ['minimal','detailed'], (
|
|
643
|
+
"mode for output processing should be either "
|
|
644
|
+
"'minimal' or 'detailed'"
|
|
645
|
+
)
|
|
646
|
+
|
|
647
|
+
labels_pred = self._inference_outputs_unprocessed[
|
|
648
|
+
'hierarchical_label_preds'
|
|
649
|
+
]
|
|
650
|
+
labels_prob = self._inference_outputs_unprocessed[
|
|
651
|
+
'probability_of_preds'
|
|
652
|
+
]
|
|
653
|
+
|
|
654
|
+
output_processing_class = OutputLabels(
|
|
655
|
+
labels_pred,
|
|
656
|
+
labels_prob,
|
|
657
|
+
self.max_depth,
|
|
658
|
+
self.num_cells
|
|
659
|
+
)
|
|
660
|
+
|
|
661
|
+
combined_labels = output_processing_class.combined_labels
|
|
662
|
+
level_zero_labels = output_processing_class.level_zero_labels
|
|
663
|
+
final_level_labels = output_processing_class.final_level_labels
|
|
664
|
+
final_level_prob = (
|
|
665
|
+
output_processing_class.final_level_prob
|
|
666
|
+
)
|
|
667
|
+
full_consistent_hierarchy = (
|
|
668
|
+
output_processing_class.full_consistent_hierarchy
|
|
669
|
+
)
|
|
670
|
+
|
|
671
|
+
self.processed_outputs = {
|
|
672
|
+
'full_hierarchical_labels': combined_labels,
|
|
673
|
+
'level_zero_labels': level_zero_labels,
|
|
674
|
+
'final_level_labels': final_level_labels,
|
|
675
|
+
'final_level_confidence': final_level_prob,
|
|
676
|
+
'full_consistent_hierarchy': full_consistent_hierarchy
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
if mode=='detailed':
|
|
680
|
+
for i in range(self.max_depth):
|
|
681
|
+
self.processed_outputs[f'level_{i+1}_labels'] = (
|
|
682
|
+
output_processing_class.all_level_labels()[i]
|
|
683
|
+
)
|
|
684
|
+
|
|
685
|
+
return self.processed_outputs
|
|
686
|
+
|
|
687
|
+
def refine_labels(self, refine_level):
|
|
688
|
+
"""
|
|
689
|
+
Refine hierarchical labels to a consistent level of granularity.
|
|
690
|
+
|
|
691
|
+
This method applies post-processing rules to standardize
|
|
692
|
+
annotations at the specified level of granularity (broad,
|
|
693
|
+
medium, or fine).
|
|
694
|
+
|
|
695
|
+
Parameters
|
|
696
|
+
----------
|
|
697
|
+
refine_level : str
|
|
698
|
+
Level of refinement: 'broad', 'medium', or 'fine'.
|
|
699
|
+
|
|
700
|
+
Returns
|
|
701
|
+
-------
|
|
702
|
+
list
|
|
703
|
+
List of refined labels at the specified level.
|
|
704
|
+
|
|
705
|
+
Raises
|
|
706
|
+
------
|
|
707
|
+
AssertionError
|
|
708
|
+
If refine_level is not valid or inference hasn't been run.
|
|
709
|
+
|
|
710
|
+
Notes
|
|
711
|
+
-----
|
|
712
|
+
For 'broad' level, this returns the top-level annotations.
|
|
713
|
+
For 'medium' and 'fine' levels, specialized refinement is
|
|
714
|
+
applied.
|
|
715
|
+
"""
|
|
716
|
+
|
|
717
|
+
assert refine_level in ['broad','medium','fine'], (
|
|
718
|
+
"refine_level should be 'broad', 'medium', or 'fine'."
|
|
719
|
+
)
|
|
720
|
+
|
|
721
|
+
assert self._inference_outputs_unprocessed is not None, (
|
|
722
|
+
"Labels can be refined only after inference model has been run."
|
|
723
|
+
)
|
|
724
|
+
|
|
725
|
+
if self._verbose:
|
|
726
|
+
print(
|
|
727
|
+
"Interpreting label predictions for consistent granularity "
|
|
728
|
+
f"at {refine_level} level.\n")
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
labels_pred = self._inference_outputs_unprocessed[
|
|
732
|
+
'hierarchical_label_preds'
|
|
733
|
+
]
|
|
734
|
+
labels_prob = self._inference_outputs_unprocessed[
|
|
735
|
+
'probability_of_preds'
|
|
736
|
+
]
|
|
737
|
+
probs = self._inference_outputs_unprocessed[
|
|
738
|
+
'softmax_vals_all'
|
|
739
|
+
]
|
|
740
|
+
|
|
741
|
+
# this conditional block is to specifically handle refinement
|
|
742
|
+
# of empty cell calls at medium and fine levels.
|
|
743
|
+
if refine_level in ['medium', 'fine']:
|
|
744
|
+
labels_pred = np.array(labels_pred, dtype=object).copy()
|
|
745
|
+
prev_labels = self._azimuth_refined_labels.get(
|
|
746
|
+
'azimuth_broad', None
|
|
747
|
+
)
|
|
748
|
+
if prev_labels is not None:
|
|
749
|
+
for i, label in enumerate(labels_pred):
|
|
750
|
+
labels_pred[i] = [
|
|
751
|
+
prev_labels[i] + l[len('Empty'):]
|
|
752
|
+
if l.startswith('Empty') else l for l in label
|
|
753
|
+
]
|
|
754
|
+
|
|
755
|
+
refine_class = PostprocessingAzimuthLabels(
|
|
756
|
+
labels_pred,
|
|
757
|
+
labels_prob,
|
|
758
|
+
self.max_depth,
|
|
759
|
+
self.num_cells,
|
|
760
|
+
probs,
|
|
761
|
+
self.inference_encoders,
|
|
762
|
+
refine_level,
|
|
763
|
+
self._model_version
|
|
764
|
+
)
|
|
765
|
+
|
|
766
|
+
results = refine_class.refine_labels()
|
|
767
|
+
|
|
768
|
+
|
|
769
|
+
(
|
|
770
|
+
self._azimuth_refined_labels[f'azimuth_{refine_level}']
|
|
771
|
+
) = results
|
|
772
|
+
|
|
773
|
+
return results
|
|
774
|
+
|
|
775
|
+
def inference_model_embeddings(self, embedding_layer_name):
|
|
776
|
+
"""
|
|
777
|
+
Extract embeddings from an intermediate layer of the inference
|
|
778
|
+
model.
|
|
779
|
+
|
|
780
|
+
Parameters
|
|
781
|
+
----------
|
|
782
|
+
embedding_layer_name : str
|
|
783
|
+
Name of the layer to extract embeddings from.
|
|
784
|
+
|
|
785
|
+
Returns
|
|
786
|
+
-------
|
|
787
|
+
numpy.ndarray
|
|
788
|
+
Embeddings from the specified layer for all query cells.
|
|
789
|
+
|
|
790
|
+
Raises
|
|
791
|
+
------
|
|
792
|
+
RuntimeError
|
|
793
|
+
If inference model is not found.
|
|
794
|
+
AssertionError
|
|
795
|
+
If input matrix has not been initialized.
|
|
796
|
+
|
|
797
|
+
Notes
|
|
798
|
+
-----
|
|
799
|
+
The embeddings are stored in the embeddings dictionary with a
|
|
800
|
+
key that combines the model name and layer name.
|
|
801
|
+
"""
|
|
802
|
+
|
|
803
|
+
if not hasattr(self, 'inference_model'):
|
|
804
|
+
raise RuntimeError("inference_model not found")
|
|
805
|
+
|
|
806
|
+
assert self._inference_input_matrix is not None, (
|
|
807
|
+
"Input matrix not initialized. Call process_query() first."
|
|
808
|
+
)
|
|
809
|
+
|
|
810
|
+
embedding_class = Embeddings(self.inference_model, embedding_layer_name)
|
|
811
|
+
|
|
812
|
+
embeddings = embedding_class.embeddings(
|
|
813
|
+
self._inference_input_matrix,
|
|
814
|
+
self._eval_batch_size
|
|
815
|
+
)
|
|
816
|
+
|
|
817
|
+
self.embeddings[
|
|
818
|
+
f'{self.inference_model_name}_{embedding_layer_name}_embed'
|
|
819
|
+
] = embeddings
|
|
820
|
+
|
|
821
|
+
return embeddings
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
def inference_model_umaps(
|
|
825
|
+
self,
|
|
826
|
+
embedding_layer_name,
|
|
827
|
+
n_neighbors=30,
|
|
828
|
+
n_components=2,
|
|
829
|
+
metric='cosine',
|
|
830
|
+
min_dist=0.3,
|
|
831
|
+
umap_lr=1.0,
|
|
832
|
+
umap_seed=42,
|
|
833
|
+
spread=1.0,
|
|
834
|
+
verbose=True,
|
|
835
|
+
init='spectral'
|
|
836
|
+
):
|
|
837
|
+
"""
|
|
838
|
+
Generate UMAP coordinates from existing embeddings.
|
|
839
|
+
|
|
840
|
+
Parameters
|
|
841
|
+
----------
|
|
842
|
+
embedding_layer_name : str
|
|
843
|
+
Name of the layer whose embeddings should be used.
|
|
844
|
+
n_neighbors : int, default=30
|
|
845
|
+
Number of neighbors for UMAP.
|
|
846
|
+
n_components : int, default=2
|
|
847
|
+
Number of dimensions for UMAP output.
|
|
848
|
+
metric : str, default='cosine'
|
|
849
|
+
Distance metric for UMAP.
|
|
850
|
+
min_dist : float, default=0.3
|
|
851
|
+
Minimum distance parameter for UMAP.
|
|
852
|
+
umap_lr : float, default=1.0
|
|
853
|
+
Learning rate for UMAP.
|
|
854
|
+
umap_seed : int, default=42
|
|
855
|
+
Random seed for reproducibility.
|
|
856
|
+
spread : float, default=1.0
|
|
857
|
+
Spread parameter for UMAP.
|
|
858
|
+
verbose : bool, default=True
|
|
859
|
+
Whether to display progress during UMAP calculation.
|
|
860
|
+
init : str, default='spectral'
|
|
861
|
+
Initialization method for UMAP.
|
|
862
|
+
|
|
863
|
+
Returns
|
|
864
|
+
-------
|
|
865
|
+
numpy.ndarray
|
|
866
|
+
UMAP coordinates for all query cells.
|
|
867
|
+
|
|
868
|
+
Raises
|
|
869
|
+
------
|
|
870
|
+
AssertionError
|
|
871
|
+
If the specified embeddings have not been generated.
|
|
872
|
+
|
|
873
|
+
Notes
|
|
874
|
+
-----
|
|
875
|
+
This method requires that embeddings have already been generated
|
|
876
|
+
using inference_model_embeddings().
|
|
877
|
+
"""
|
|
878
|
+
|
|
879
|
+
embedding_key_base = (
|
|
880
|
+
f"{self.inference_model_name}_{embedding_layer_name}"
|
|
881
|
+
)
|
|
882
|
+
embedding_key = embedding_key_base + '_embed'
|
|
883
|
+
umap_key = embedding_key_base + '_umap'
|
|
884
|
+
|
|
885
|
+
assert embedding_key in self.embeddings.keys(), (
|
|
886
|
+
f"Embedding '{embedding_key}' not found. Generate embeddings "
|
|
887
|
+
"before creating umap."
|
|
888
|
+
)
|
|
889
|
+
|
|
890
|
+
umap_class = Umaps(
|
|
891
|
+
n_neighbors=n_neighbors,
|
|
892
|
+
n_components=n_components,
|
|
893
|
+
metric=metric,
|
|
894
|
+
min_dist=min_dist,
|
|
895
|
+
umap_lr=umap_lr,
|
|
896
|
+
umap_seed=umap_seed,
|
|
897
|
+
spread=spread,
|
|
898
|
+
verbose=verbose,
|
|
899
|
+
init=init
|
|
900
|
+
)
|
|
901
|
+
|
|
902
|
+
umap_gen = umap_class.create_umap(self.embeddings[embedding_key])
|
|
903
|
+
self.umaps[umap_key] = umap_gen
|
|
904
|
+
|
|
905
|
+
return umap_gen
|
|
906
|
+
|
|
907
|
+
def inference_embeddings_and_umap(
|
|
908
|
+
self,
|
|
909
|
+
embedding_layer_name,
|
|
910
|
+
n_neighbors=30,
|
|
911
|
+
n_components=2,
|
|
912
|
+
metric='cosine',
|
|
913
|
+
min_dist=0.3,
|
|
914
|
+
umap_lr=1.0,
|
|
915
|
+
umap_seed=42,
|
|
916
|
+
spread=1.0,
|
|
917
|
+
verbose=True,
|
|
918
|
+
init='spectral'
|
|
919
|
+
):
|
|
920
|
+
"""
|
|
921
|
+
Generate both embeddings and UMAP coordinates in one operation.
|
|
922
|
+
|
|
923
|
+
This is a convenience method that combines the functionality of
|
|
924
|
+
inference_model_embeddings() and inference_model_umaps().
|
|
925
|
+
|
|
926
|
+
Parameters
|
|
927
|
+
----------
|
|
928
|
+
embedding_layer_name : str
|
|
929
|
+
Name of the layer to extract embeddings from.
|
|
930
|
+
n_neighbors : int, default=30
|
|
931
|
+
Number of neighbors for UMAP.
|
|
932
|
+
n_components : int, default=2
|
|
933
|
+
Number of dimensions for UMAP output.
|
|
934
|
+
metric : str, default='cosine'
|
|
935
|
+
Distance metric for UMAP.
|
|
936
|
+
min_dist : float, default=0.3
|
|
937
|
+
Minimum distance parameter for UMAP.
|
|
938
|
+
umap_lr : float, default=1.0
|
|
939
|
+
Learning rate for UMAP.
|
|
940
|
+
umap_seed : int, default=42
|
|
941
|
+
Random seed for reproducibility.
|
|
942
|
+
spread : float, default=1.0
|
|
943
|
+
Spread parameter for UMAP.
|
|
944
|
+
verbose : bool, default=True
|
|
945
|
+
Whether to display progress during UMAP calculation.
|
|
946
|
+
init : str, default='spectral'
|
|
947
|
+
Initialization method for UMAP.
|
|
948
|
+
|
|
949
|
+
Returns
|
|
950
|
+
-------
|
|
951
|
+
tuple
|
|
952
|
+
Tuple containing (embeddings, umap_coordinates).
|
|
953
|
+
|
|
954
|
+
Raises
|
|
955
|
+
------
|
|
956
|
+
RuntimeError
|
|
957
|
+
If inference model is not found.
|
|
958
|
+
ValueError
|
|
959
|
+
If input matrix has not been initialized.
|
|
960
|
+
|
|
961
|
+
Notes
|
|
962
|
+
-----
|
|
963
|
+
This method may be more efficient than calling the two component
|
|
964
|
+
methods separately, depending on usage, as it avoids storing
|
|
965
|
+
intermediate results in memory twice.
|
|
966
|
+
"""
|
|
967
|
+
|
|
968
|
+
if not hasattr(self, 'inference_model'):
|
|
969
|
+
raise RuntimeError("inference_model not found")
|
|
970
|
+
|
|
971
|
+
if self._inference_input_matrix is None:
|
|
972
|
+
raise ValueError(
|
|
973
|
+
"X_query has not been processed for extraction of embeddings.\n"
|
|
974
|
+
"Run process_query() first."
|
|
975
|
+
)
|
|
976
|
+
|
|
977
|
+
embeddings_and_umap_class = EmbeddingsAndUmap(
|
|
978
|
+
self.inference_model,
|
|
979
|
+
embedding_layer_name,
|
|
980
|
+
n_neighbors=n_neighbors,
|
|
981
|
+
n_components=n_components,
|
|
982
|
+
metric=metric,
|
|
983
|
+
min_dist=min_dist,
|
|
984
|
+
umap_lr=umap_lr,
|
|
985
|
+
umap_seed=umap_seed,
|
|
986
|
+
spread=spread,
|
|
987
|
+
verbose=verbose,
|
|
988
|
+
init=init
|
|
989
|
+
)
|
|
990
|
+
|
|
991
|
+
em, umap_em = embeddings_and_umap_class.create_embeddings_and_umap(
|
|
992
|
+
self._inference_input_matrix,
|
|
993
|
+
self._eval_batch_size
|
|
994
|
+
)
|
|
995
|
+
|
|
996
|
+
embed_key = f'{self.inference_model_name}_{embedding_layer_name}_embed'
|
|
997
|
+
umap_key = f'{self.inference_model_name}_{embedding_layer_name}_umap'
|
|
998
|
+
|
|
999
|
+
self.embeddings[embed_key] = em
|
|
1000
|
+
self.umaps[umap_key] = umap_em
|
|
1001
|
+
|
|
1002
|
+
return em, umap_em
|
|
1003
|
+
|
|
1004
|
+
def update_cells_meta(self):
|
|
1005
|
+
"""
|
|
1006
|
+
Update cells_meta DataFrame with inference results.
|
|
1007
|
+
|
|
1008
|
+
This method adds or updates columns in the cell metadata DataFrame
|
|
1009
|
+
with inference results, including predictions and refined labels.
|
|
1010
|
+
|
|
1011
|
+
Returns
|
|
1012
|
+
-------
|
|
1013
|
+
pandas.DataFrame
|
|
1014
|
+
Updated cell metadata DataFrame.
|
|
1015
|
+
|
|
1016
|
+
Raises
|
|
1017
|
+
------
|
|
1018
|
+
TypeError
|
|
1019
|
+
If cells_meta is not a pandas DataFrame.
|
|
1020
|
+
RuntimeError
|
|
1021
|
+
If num_cells attribute is not set.
|
|
1022
|
+
ValueError
|
|
1023
|
+
If the number of values doesn't match the number of cells.
|
|
1024
|
+
|
|
1025
|
+
Notes
|
|
1026
|
+
-----
|
|
1027
|
+
If both broad level annotations and level_zero_labels exist,
|
|
1028
|
+
the latter is dropped to avoid duplication.
|
|
1029
|
+
"""
|
|
1030
|
+
|
|
1031
|
+
if not isinstance(self.cells_meta, pd.DataFrame):
|
|
1032
|
+
raise TypeError(
|
|
1033
|
+
"Query cell meta is not available as pandas dataframe"
|
|
1034
|
+
)
|
|
1035
|
+
|
|
1036
|
+
if not hasattr(self, 'num_cells'):
|
|
1037
|
+
raise RuntimeError("num_cells not found")
|
|
1038
|
+
|
|
1039
|
+
if self.processed_outputs and len(self.processed_outputs) > 0:
|
|
1040
|
+
for col_idx, (meta_col, values) in enumerate(
|
|
1041
|
+
self.processed_outputs.items()
|
|
1042
|
+
):
|
|
1043
|
+
if len(values) != self.num_cells:
|
|
1044
|
+
raise ValueError(
|
|
1045
|
+
f"Column {meta_col} has {len(values)} values but "
|
|
1046
|
+
f"expected {self.num_cells}"
|
|
1047
|
+
)
|
|
1048
|
+
self.cells_meta = insert_col(
|
|
1049
|
+
self.cells_meta,
|
|
1050
|
+
col_idx,
|
|
1051
|
+
meta_col,
|
|
1052
|
+
values
|
|
1053
|
+
)
|
|
1054
|
+
|
|
1055
|
+
if (
|
|
1056
|
+
len(self._azimuth_refined_labels)
|
|
1057
|
+
) > 0:
|
|
1058
|
+
start_idx = len(self.processed_outputs or [])
|
|
1059
|
+
for col_idx, (refine_col, values) in enumerate(
|
|
1060
|
+
self._azimuth_refined_labels.items(), start=start_idx
|
|
1061
|
+
):
|
|
1062
|
+
if len(values) != self.num_cells:
|
|
1063
|
+
raise ValueError(
|
|
1064
|
+
f"Column {refine_col} has {len(values)} values but "
|
|
1065
|
+
f"expected {self.num_cells}"
|
|
1066
|
+
)
|
|
1067
|
+
self.cells_meta = insert_col(
|
|
1068
|
+
self.cells_meta,
|
|
1069
|
+
col_idx,
|
|
1070
|
+
refine_col,
|
|
1071
|
+
values
|
|
1072
|
+
)
|
|
1073
|
+
|
|
1074
|
+
if (
|
|
1075
|
+
(
|
|
1076
|
+
"azimuth_broad" in self.cells_meta.keys()
|
|
1077
|
+
) and (
|
|
1078
|
+
"level_zero_labels" in self.cells_meta.keys()
|
|
1079
|
+
)
|
|
1080
|
+
):
|
|
1081
|
+
self.cells_meta.drop('level_zero_labels', axis=1, inplace=True)
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
return self.cells_meta
|
|
1085
|
+
|
|
1086
|
+
|
|
1087
|
+
@contextmanager
|
|
1088
|
+
def _scoped_slice(self, start, end):
|
|
1089
|
+
"""
|
|
1090
|
+
Context manager that temporarily scopes this object to a
|
|
1091
|
+
slice of cells for minibatched processing.
|
|
1092
|
+
|
|
1093
|
+
Saves and restores instance state so that existing methods
|
|
1094
|
+
(run_inference_model, calibrate_predictions, process_outputs,
|
|
1095
|
+
refine_labels) operate on the slice as if it were the full
|
|
1096
|
+
dataset. On exit, original state is restored and intermediate
|
|
1097
|
+
outputs from the slice are discarded.
|
|
1098
|
+
|
|
1099
|
+
Parameters
|
|
1100
|
+
----------
|
|
1101
|
+
start : int
|
|
1102
|
+
Start index (inclusive) into the inference input matrix.
|
|
1103
|
+
end : int
|
|
1104
|
+
End index (exclusive) into the inference input matrix.
|
|
1105
|
+
|
|
1106
|
+
Yields
|
|
1107
|
+
------
|
|
1108
|
+
None
|
|
1109
|
+
|
|
1110
|
+
Notes
|
|
1111
|
+
-----
|
|
1112
|
+
This is a private method intended for use by subclasses that
|
|
1113
|
+
implement minibatched pipelines. It does not affect embeddings
|
|
1114
|
+
or umaps state.
|
|
1115
|
+
"""
|
|
1116
|
+
orig_inference_input_matrix = self._inference_input_matrix
|
|
1117
|
+
orig_num_cells = self.num_cells
|
|
1118
|
+
orig_inference_outputs = self._inference_outputs_unprocessed
|
|
1119
|
+
orig_processed_outputs = self.processed_outputs
|
|
1120
|
+
orig_refined_labels = self._azimuth_refined_labels
|
|
1121
|
+
|
|
1122
|
+
self._inference_input_matrix = orig_inference_input_matrix[start:end]
|
|
1123
|
+
self.num_cells = end - start
|
|
1124
|
+
self._inference_outputs_unprocessed = None
|
|
1125
|
+
self.processed_outputs = None
|
|
1126
|
+
self._azimuth_refined_labels = {}
|
|
1127
|
+
|
|
1128
|
+
try:
|
|
1129
|
+
yield
|
|
1130
|
+
finally:
|
|
1131
|
+
self._inference_input_matrix = orig_inference_input_matrix
|
|
1132
|
+
self.num_cells = orig_num_cells
|
|
1133
|
+
self._inference_outputs_unprocessed = orig_inference_outputs
|
|
1134
|
+
self.processed_outputs = orig_processed_outputs
|
|
1135
|
+
self._azimuth_refined_labels = orig_refined_labels
|
|
1136
|
+
|
|
1137
|
+
|
|
1138
|
+
def pack_adata(self, save_path = None):
|
|
1139
|
+
"""
|
|
1140
|
+
Create an AnnData object with all results and optionally save to
|
|
1141
|
+
disk.
|
|
1142
|
+
|
|
1143
|
+
This method packages all results (expression data, metadata,
|
|
1144
|
+
embeddings, and UMAP coordinates) into a unified AnnData object
|
|
1145
|
+
for further analysis or visualization.
|
|
1146
|
+
|
|
1147
|
+
Parameters
|
|
1148
|
+
----------
|
|
1149
|
+
save_path : str, optional
|
|
1150
|
+
Path to save the AnnData object as an H5AD file.
|
|
1151
|
+
If None, the object is created but not saved.
|
|
1152
|
+
|
|
1153
|
+
Returns
|
|
1154
|
+
-------
|
|
1155
|
+
anndata.AnnData
|
|
1156
|
+
AnnData object containing all query data and results.
|
|
1157
|
+
|
|
1158
|
+
Notes
|
|
1159
|
+
-----
|
|
1160
|
+
If the specified save_path already exists, a timestamp is
|
|
1161
|
+
appended to the filename to prevent overwriting.
|
|
1162
|
+
Metadata is automatically coerced to h5ad-compatible types.
|
|
1163
|
+
"""
|
|
1164
|
+
|
|
1165
|
+
|
|
1166
|
+
cells_meta_coerced = coerce_metadata_types(self.cells_meta)
|
|
1167
|
+
features_meta_coerced = coerce_metadata_types(self.features_meta)
|
|
1168
|
+
|
|
1169
|
+
|
|
1170
|
+
all_embeddings = {**self.embeddings, **self.umaps}
|
|
1171
|
+
|
|
1172
|
+
adata_obj = create_anndata(
|
|
1173
|
+
self.X_query,
|
|
1174
|
+
cells_meta_coerced,
|
|
1175
|
+
features_meta_coerced,
|
|
1176
|
+
embeddings = all_embeddings
|
|
1177
|
+
)
|
|
1178
|
+
|
|
1179
|
+
if save_path:
|
|
1180
|
+
if os.path.exists(save_path):
|
|
1181
|
+
|
|
1182
|
+
base_path, ext = os.path.splitext(save_path)
|
|
1183
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
1184
|
+
save_path = f"{base_path}_{timestamp}{ext}"
|
|
1185
|
+
print(
|
|
1186
|
+
f"File {save_path} already exists. Adding timestamp "
|
|
1187
|
+
"suffix to prevent overwrite.",
|
|
1188
|
+
)
|
|
1189
|
+
print(f"Saving to: {save_path}")
|
|
1190
|
+
adata_obj.write(save_path)
|
|
1191
|
+
|
|
1192
|
+
self.query = adata_obj
|
|
1193
|
+
|
|
1194
|
+
return adata_obj
|
|
1195
|
+
|
|
1196
|
+
def map_to_cell_ontology(
|
|
1197
|
+
self,
|
|
1198
|
+
src_col,
|
|
1199
|
+
include_cl_id=False
|
|
1200
|
+
):
|
|
1201
|
+
"""
|
|
1202
|
+
Map annotation labels in cells_meta to Cell Ontology terms.
|
|
1203
|
+
|
|
1204
|
+
Applies the versioned cell ontology map for the model version
|
|
1205
|
+
used in this run to the specified column in cells_meta, adding
|
|
1206
|
+
one (or two) new columns immediately after src_col. Unmapped
|
|
1207
|
+
labels are set to 'unmapped' and a single warning is emitted
|
|
1208
|
+
listing all unique labels that could not be mapped.
|
|
1209
|
+
|
|
1210
|
+
Parameters
|
|
1211
|
+
----------
|
|
1212
|
+
src_col : str
|
|
1213
|
+
Name of the column in cells_meta carrying the source
|
|
1214
|
+
annotation labels to be mapped.
|
|
1215
|
+
include_cl_id : bool, default False
|
|
1216
|
+
If True, also adds a column named {src_col}_CL_ID
|
|
1217
|
+
containing the CL identifier string (e.g. 'CL:0000236'),
|
|
1218
|
+
or 'unmapped'.
|
|
1219
|
+
|
|
1220
|
+
Returns
|
|
1221
|
+
-------
|
|
1222
|
+
pandas.DataFrame
|
|
1223
|
+
Updated cells_meta with new column(s) added.
|
|
1224
|
+
|
|
1225
|
+
New columns added
|
|
1226
|
+
-----------------
|
|
1227
|
+
{src_col}_CL
|
|
1228
|
+
CL label string for each cell, or 'unmapped'.
|
|
1229
|
+
{src_col}_CL_ID (only if include_cl_id=True)
|
|
1230
|
+
CL identifier string, or 'unmapped'.
|
|
1231
|
+
|
|
1232
|
+
Raises
|
|
1233
|
+
------
|
|
1234
|
+
TypeError
|
|
1235
|
+
If cells_meta is not a pandas DataFrame.
|
|
1236
|
+
ValueError
|
|
1237
|
+
If src_col is not present in cells_meta.
|
|
1238
|
+
|
|
1239
|
+
Warns
|
|
1240
|
+
-----
|
|
1241
|
+
UserWarning
|
|
1242
|
+
Emitted once, listing all unique labels that could not be
|
|
1243
|
+
mapped to a CL term.
|
|
1244
|
+
|
|
1245
|
+
Examples
|
|
1246
|
+
--------
|
|
1247
|
+
>>> azimuth = AzimuthNN(adata)
|
|
1248
|
+
>>> azimuth.map_to_cell_ontology('azimuth_fine')
|
|
1249
|
+
>>> azimuth.map_to_cell_ontology('azimuth_broad', include_cl_id=True)
|
|
1250
|
+
"""
|
|
1251
|
+
if not isinstance(self.cells_meta, pd.DataFrame):
|
|
1252
|
+
raise TypeError(
|
|
1253
|
+
"cells_meta is not available as a pandas DataFrame."
|
|
1254
|
+
)
|
|
1255
|
+
|
|
1256
|
+
self.cells_meta = map_to_cell_ontology(
|
|
1257
|
+
self.cells_meta,
|
|
1258
|
+
src_col=src_col,
|
|
1259
|
+
model_version=self._model_version,
|
|
1260
|
+
include_cl_id=include_cl_id
|
|
1261
|
+
)
|
|
1262
|
+
|
|
1263
|
+
return self.cells_meta
|
|
1264
|
+
|
|
1265
|
+
|
|
1266
|
+
|
|
1267
|
+
|
|
1268
|
+
########################################################################
|
|
1269
|
+
######## Object for high level interactive usage #######################
|
|
1270
|
+
|
|
1271
|
+
_VALID_REFINE_LEVELS = ['broad', 'medium', 'fine']
|
|
1272
|
+
|
|
1273
|
+
class AzimuthNN(AzimuthNN_base):
|
|
1274
|
+
"""
|
|
1275
|
+
AzimuthNN: A high-level interface for a cell annotation pipeline
|
|
1276
|
+
based on the Azimuth neural network.
|
|
1277
|
+
|
|
1278
|
+
This class wraps around the AzimuthNN_base class to provide a
|
|
1279
|
+
simplified workflow for hierarchical cell type annotation based on
|
|
1280
|
+
single-cell RNA-seq data, handling data loading, preprocessing,
|
|
1281
|
+
model inference, and visualization in a streamlined manner.
|
|
1282
|
+
|
|
1283
|
+
The pipeline automatically applies confidence calibration to improve
|
|
1284
|
+
the reliability of prediction confidence scores using trained
|
|
1285
|
+
calibration models.
|
|
1286
|
+
|
|
1287
|
+
As of v0.3.0, inference, calibration, output processing, and label
|
|
1288
|
+
refinement are performed in minibatches to reduce peak memory usage.
|
|
1289
|
+
Softmax arrays and intermediate inference outputs are released after
|
|
1290
|
+
each minibatch is processed.
|
|
1291
|
+
|
|
1292
|
+
For more fine-grained control over the annotation process, users
|
|
1293
|
+
should directly use the AzimuthNN_base class.
|
|
1294
|
+
|
|
1295
|
+
Parameters
|
|
1296
|
+
----------
|
|
1297
|
+
query_arg : Union[str, anndata.AnnData]
|
|
1298
|
+
Either an AnnData object containing single-cell data or a path
|
|
1299
|
+
to an h5ad file.
|
|
1300
|
+
feature_names_col : str, optional
|
|
1301
|
+
Column in the anndata_object.var dataframe that contains the
|
|
1302
|
+
gene names to use for model input. If None, assumes var_names
|
|
1303
|
+
are already the correct gene identifiers.
|
|
1304
|
+
annotation_pipeline : str, default='supervised'
|
|
1305
|
+
Type of annotation pipeline to use for cell type prediction.
|
|
1306
|
+
model_version: str, default set to match package major version.
|
|
1307
|
+
Model version to use.
|
|
1308
|
+
eval_batch_size : int, default=8192
|
|
1309
|
+
Batch size to use during model inference and minibatched
|
|
1310
|
+
annotation pipeline.
|
|
1311
|
+
normalization_override : bool, default=False
|
|
1312
|
+
If True, skips normalization check and forces processing to
|
|
1313
|
+
continue.
|
|
1314
|
+
norm_check_batch_size : int, default=1000
|
|
1315
|
+
Number of cells to sample for normalization check.
|
|
1316
|
+
output_mode : str, default='minimal'
|
|
1317
|
+
Controls the verbosity of output in the cell meta dataframe.
|
|
1318
|
+
Options are 'minimal' or 'detailed'.
|
|
1319
|
+
refine : list, bool, or None, default=True
|
|
1320
|
+
Which refinement levels to apply during initialization.
|
|
1321
|
+
Accepts a list containing one or more of 'broad', 'medium',
|
|
1322
|
+
'fine'. If 'medium' or 'fine' is included without 'broad',
|
|
1323
|
+
'broad' is added automatically as it is a prerequisite.
|
|
1324
|
+
Also accepts True (equivalent to ['broad', 'medium', 'fine']),
|
|
1325
|
+
False or None (no refinement), or an empty list (no refinement).
|
|
1326
|
+
As of v0.3.0, refinement is performed by default within the
|
|
1327
|
+
minibatched pipeline.
|
|
1328
|
+
|
|
1329
|
+
Attributes
|
|
1330
|
+
----------
|
|
1331
|
+
cells_meta : pandas.DataFrame or None
|
|
1332
|
+
Cell metadata.
|
|
1333
|
+
embeddings : dict
|
|
1334
|
+
Contains embeddings extracted from the model.
|
|
1335
|
+
umaps : dict
|
|
1336
|
+
Contains UMAP projections of the embeddings.
|
|
1337
|
+
|
|
1338
|
+
Raises
|
|
1339
|
+
------
|
|
1340
|
+
TypeError
|
|
1341
|
+
If query_arg is not an AnnData object or a string path,
|
|
1342
|
+
if normalization_override is not a bool,
|
|
1343
|
+
or if norm_check_batch_size is not an integer.
|
|
1344
|
+
ValueError
|
|
1345
|
+
If output_mode is not 'minimal' or 'detailed', or if refine
|
|
1346
|
+
contains invalid refinement levels.
|
|
1347
|
+
|
|
1348
|
+
Examples
|
|
1349
|
+
--------
|
|
1350
|
+
>>> import anndata
|
|
1351
|
+
>>> adata = anndata.read_h5ad('my_data.h5ad')
|
|
1352
|
+
>>> azimuth = AzimuthNN(adata)
|
|
1353
|
+
>>> embeddings = azimuth.azimuth_embed()
|
|
1354
|
+
>>> umap = azimuth.azimuth_umap()
|
|
1355
|
+
>>> cell_metadata = azimuth.cells_meta
|
|
1356
|
+
|
|
1357
|
+
Refine only at broad and fine levels:
|
|
1358
|
+
>>> azimuth = AzimuthNN(adata, refine=['broad', 'fine'])
|
|
1359
|
+
|
|
1360
|
+
Skip refinement entirely:
|
|
1361
|
+
>>> azimuth = AzimuthNN(adata, refine=False)
|
|
1362
|
+
"""
|
|
1363
|
+
|
|
1364
|
+
def __init__(
|
|
1365
|
+
self,
|
|
1366
|
+
query_arg,
|
|
1367
|
+
feature_names_col=None,
|
|
1368
|
+
annotation_pipeline='supervised',
|
|
1369
|
+
model_version=model_version_default,
|
|
1370
|
+
eval_batch_size=8192,
|
|
1371
|
+
normalization_override=False,
|
|
1372
|
+
norm_check_batch_size=100,
|
|
1373
|
+
output_mode='minimal',
|
|
1374
|
+
refine=True
|
|
1375
|
+
):
|
|
1376
|
+
|
|
1377
|
+
"""
|
|
1378
|
+
Initialize AzimuthNN with automatic annotation pipeline execution.
|
|
1379
|
+
|
|
1380
|
+
This constructor automatically runs the complete annotation
|
|
1381
|
+
workflow in minibatches:
|
|
1382
|
+
1. Data loading and preprocessing
|
|
1383
|
+
2. For each minibatch:
|
|
1384
|
+
a. Model inference
|
|
1385
|
+
b. Confidence calibration using trained calibration models
|
|
1386
|
+
c. Output processing
|
|
1387
|
+
d. Label refinement (per refine parameter)
|
|
1388
|
+
e. Release of softmax arrays and intermediate outputs
|
|
1389
|
+
3. Accumulate results and update cell metadata
|
|
1390
|
+
"""
|
|
1391
|
+
|
|
1392
|
+
if (
|
|
1393
|
+
not isinstance(query_arg, str) and
|
|
1394
|
+
not isinstance(query_arg, anndata.AnnData)
|
|
1395
|
+
):
|
|
1396
|
+
raise TypeError(
|
|
1397
|
+
"query argument must either be AnnData object or "
|
|
1398
|
+
"path to an h5ad file."
|
|
1399
|
+
)
|
|
1400
|
+
|
|
1401
|
+
if feature_names_col is None:
|
|
1402
|
+
warnings.warn(
|
|
1403
|
+
"Ensure that the features metadata is indexed "
|
|
1404
|
+
"with gene names.",
|
|
1405
|
+
UserWarning
|
|
1406
|
+
)
|
|
1407
|
+
|
|
1408
|
+
if not isinstance(normalization_override, bool):
|
|
1409
|
+
raise TypeError("normalization override must be a bool")
|
|
1410
|
+
|
|
1411
|
+
if not isinstance(norm_check_batch_size, int):
|
|
1412
|
+
raise TypeError("norm_check_batch_size must be an integer")
|
|
1413
|
+
|
|
1414
|
+
if output_mode not in ['minimal', 'detailed']:
|
|
1415
|
+
raise ValueError(
|
|
1416
|
+
"output_mode must be either 'minimal' or 'detailed'"
|
|
1417
|
+
)
|
|
1418
|
+
|
|
1419
|
+
self._refine_levels = self._parse_refine_arg(refine)
|
|
1420
|
+
|
|
1421
|
+
self._query_arg = query_arg
|
|
1422
|
+
self._normalization_override = normalization_override
|
|
1423
|
+
self._norm_check_batch_size = norm_check_batch_size
|
|
1424
|
+
self._output_mode = output_mode
|
|
1425
|
+
|
|
1426
|
+
super().__init__(
|
|
1427
|
+
annotation_pipeline,
|
|
1428
|
+
model_version,
|
|
1429
|
+
eval_batch_size
|
|
1430
|
+
)
|
|
1431
|
+
|
|
1432
|
+
if isinstance(self._query_arg, anndata.AnnData):
|
|
1433
|
+
self.query_adata(
|
|
1434
|
+
self._query_arg,
|
|
1435
|
+
feature_names_col = feature_names_col
|
|
1436
|
+
)
|
|
1437
|
+
elif isinstance(self._query_arg, str):
|
|
1438
|
+
self.query_h5ad(
|
|
1439
|
+
self._query_arg,
|
|
1440
|
+
feature_names_col = feature_names_col
|
|
1441
|
+
)
|
|
1442
|
+
|
|
1443
|
+
|
|
1444
|
+
|
|
1445
|
+
self.process_query(
|
|
1446
|
+
normalization_override = self._normalization_override,
|
|
1447
|
+
norm_check_batch_size = self._norm_check_batch_size
|
|
1448
|
+
)
|
|
1449
|
+
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
self._run_minibatched_pipeline()
|
|
1453
|
+
|
|
1454
|
+
|
|
1455
|
+
@staticmethod
|
|
1456
|
+
def _parse_refine_arg(refine):
|
|
1457
|
+
"""
|
|
1458
|
+
Parse and validate the refine argument into an ordered list of
|
|
1459
|
+
refinement levels.
|
|
1460
|
+
|
|
1461
|
+
Parameters
|
|
1462
|
+
----------
|
|
1463
|
+
refine : list, bool, or None
|
|
1464
|
+
Raw refine argument from __init__.
|
|
1465
|
+
|
|
1466
|
+
Returns
|
|
1467
|
+
-------
|
|
1468
|
+
list
|
|
1469
|
+
Ordered list of refinement levels to apply. May be empty
|
|
1470
|
+
if no refinement is requested.
|
|
1471
|
+
|
|
1472
|
+
Raises
|
|
1473
|
+
------
|
|
1474
|
+
TypeError
|
|
1475
|
+
If refine is not a list, bool, or None.
|
|
1476
|
+
ValueError
|
|
1477
|
+
If refine contains invalid refinement level strings.
|
|
1478
|
+
"""
|
|
1479
|
+
|
|
1480
|
+
if refine is None or refine is False:
|
|
1481
|
+
return []
|
|
1482
|
+
|
|
1483
|
+
if refine is True:
|
|
1484
|
+
return ['broad', 'medium', 'fine']
|
|
1485
|
+
|
|
1486
|
+
if not isinstance(refine, list):
|
|
1487
|
+
raise TypeError(
|
|
1488
|
+
"refine must be a list of refinement levels, "
|
|
1489
|
+
"True, False, or None."
|
|
1490
|
+
)
|
|
1491
|
+
|
|
1492
|
+
if len(refine) == 0:
|
|
1493
|
+
return []
|
|
1494
|
+
|
|
1495
|
+
invalid = [
|
|
1496
|
+
level for level in refine
|
|
1497
|
+
if level not in _VALID_REFINE_LEVELS
|
|
1498
|
+
]
|
|
1499
|
+
if invalid:
|
|
1500
|
+
raise ValueError(
|
|
1501
|
+
f"Invalid refinement level(s): {invalid}. "
|
|
1502
|
+
f"Valid options are {_VALID_REFINE_LEVELS}."
|
|
1503
|
+
)
|
|
1504
|
+
|
|
1505
|
+
# broad is a trivial prerequisite for medium and fine
|
|
1506
|
+
if ('medium' in refine or 'fine' in refine) and (
|
|
1507
|
+
'broad' not in refine
|
|
1508
|
+
):
|
|
1509
|
+
refine = ['broad'] + refine
|
|
1510
|
+
|
|
1511
|
+
# enforce canonical ordering
|
|
1512
|
+
ordered = [
|
|
1513
|
+
level for level in _VALID_REFINE_LEVELS
|
|
1514
|
+
if level in refine
|
|
1515
|
+
]
|
|
1516
|
+
|
|
1517
|
+
return ordered
|
|
1518
|
+
|
|
1519
|
+
|
|
1520
|
+
def _run_minibatched_pipeline(self):
|
|
1521
|
+
"""
|
|
1522
|
+
Run inference, calibration, output processing, and optional
|
|
1523
|
+
refinement in minibatches. After each minibatch, intermediate
|
|
1524
|
+
softmax arrays and unprocessed outputs are released via the
|
|
1525
|
+
_scoped_slice context manager on AzimuthNN_base.
|
|
1526
|
+
|
|
1527
|
+
Internal print messages from run_inference_model,
|
|
1528
|
+
calibrate_predictions, and refine_labels are suppressed
|
|
1529
|
+
during minibatched processing.
|
|
1530
|
+
"""
|
|
1531
|
+
|
|
1532
|
+
n_cells = self._inference_input_matrix.shape[0]
|
|
1533
|
+
n_batches = (
|
|
1534
|
+
(n_cells + self._eval_batch_size - 1)
|
|
1535
|
+
// self._eval_batch_size
|
|
1536
|
+
)
|
|
1537
|
+
|
|
1538
|
+
if self._refine_levels:
|
|
1539
|
+
refine_str = ', '.join(self._refine_levels)
|
|
1540
|
+
else:
|
|
1541
|
+
refine_str = 'none'
|
|
1542
|
+
|
|
1543
|
+
print(
|
|
1544
|
+
f"Splitting query data into {n_batches} "
|
|
1545
|
+
f"evaluation batch(es) of up to \n"
|
|
1546
|
+
f"{self._eval_batch_size} cells.\n"
|
|
1547
|
+
)
|
|
1548
|
+
if refine_str != 'none':
|
|
1549
|
+
print(
|
|
1550
|
+
"Interpreting label predictions for consistent granularity "
|
|
1551
|
+
f"at {refine_str} level(s).\n"
|
|
1552
|
+
)
|
|
1553
|
+
|
|
1554
|
+
# accumulators for results across minibatches
|
|
1555
|
+
all_processed_outputs = {}
|
|
1556
|
+
all_refined_labels = {}
|
|
1557
|
+
|
|
1558
|
+
# suppress internal prints during minibatched processing
|
|
1559
|
+
self._verbose = False
|
|
1560
|
+
|
|
1561
|
+
print("Running model:")
|
|
1562
|
+
|
|
1563
|
+
for batch_idx in range(n_batches):
|
|
1564
|
+
start = batch_idx * self._eval_batch_size
|
|
1565
|
+
end = min(
|
|
1566
|
+
(batch_idx + 1) * self._eval_batch_size, n_cells
|
|
1567
|
+
)
|
|
1568
|
+
|
|
1569
|
+
with MemoryContext():
|
|
1570
|
+
with self._scoped_slice(start, end):
|
|
1571
|
+
|
|
1572
|
+
_ = self.run_inference_model()
|
|
1573
|
+
_ = self.calibrate_predictions()
|
|
1574
|
+
_ = self.process_outputs(mode=self._output_mode)
|
|
1575
|
+
|
|
1576
|
+
for level in self._refine_levels:
|
|
1577
|
+
_ = self.refine_labels(refine_level=level)
|
|
1578
|
+
|
|
1579
|
+
# collect before scope restores state
|
|
1580
|
+
for key, values in (
|
|
1581
|
+
self.processed_outputs.items()
|
|
1582
|
+
):
|
|
1583
|
+
if key not in all_processed_outputs:
|
|
1584
|
+
all_processed_outputs[key] = (
|
|
1585
|
+
[None] * n_cells
|
|
1586
|
+
)
|
|
1587
|
+
all_processed_outputs[key][start:end] = (
|
|
1588
|
+
list(values)
|
|
1589
|
+
)
|
|
1590
|
+
|
|
1591
|
+
for key, values in (
|
|
1592
|
+
self._azimuth_refined_labels.items()
|
|
1593
|
+
):
|
|
1594
|
+
if key not in all_refined_labels:
|
|
1595
|
+
all_refined_labels[key] = (
|
|
1596
|
+
[None] * n_cells
|
|
1597
|
+
)
|
|
1598
|
+
all_refined_labels[key][start:end] = (
|
|
1599
|
+
list(values)
|
|
1600
|
+
)
|
|
1601
|
+
|
|
1602
|
+
# release minibatch softmax arrays and
|
|
1603
|
+
# intermediate outputs before scope restores
|
|
1604
|
+
self._inference_outputs_unprocessed = None
|
|
1605
|
+
|
|
1606
|
+
self._verbose = True
|
|
1607
|
+
|
|
1608
|
+
# set final accumulated results
|
|
1609
|
+
self.processed_outputs = all_processed_outputs
|
|
1610
|
+
self._azimuth_refined_labels = all_refined_labels
|
|
1611
|
+
self.annotations = self.processed_outputs
|
|
1612
|
+
|
|
1613
|
+
_ = self.update_cells_meta()
|
|
1614
|
+
|
|
1615
|
+
|
|
1616
|
+
def azimuth_refine(self, refine=None):
|
|
1617
|
+
"""
|
|
1618
|
+
Refine cell type annotations at multiple granularity levels.
|
|
1619
|
+
|
|
1620
|
+
.. deprecated:: 0.3.0
|
|
1621
|
+
Label refinement is now performed during initialization as
|
|
1622
|
+
part of the minibatched pipeline. This method is retained
|
|
1623
|
+
for backwards compatibility and will be removed in a future
|
|
1624
|
+
release. Use the ``refine`` parameter in :class:`AzimuthNN`
|
|
1625
|
+
initialization instead.
|
|
1626
|
+
|
|
1627
|
+
If a refinement level was not included at initialization,
|
|
1628
|
+
re-initialization is required since softmax arrays are no
|
|
1629
|
+
longer held in memory after the pipeline completes.
|
|
1630
|
+
|
|
1631
|
+
Parameters
|
|
1632
|
+
----------
|
|
1633
|
+
refine : list, optional
|
|
1634
|
+
List of refinement levels to check. If None, checks all
|
|
1635
|
+
three levels ['broad', 'medium', 'fine'].
|
|
1636
|
+
"""
|
|
1637
|
+
|
|
1638
|
+
warnings.warn(
|
|
1639
|
+
"azimuth_refine() is deprecated as of v0.3.0 and will be "
|
|
1640
|
+
"removed in a future release. Use the 'refine' parameter "
|
|
1641
|
+
"in AzimuthNN initialization instead.",
|
|
1642
|
+
DeprecationWarning,
|
|
1643
|
+
stacklevel=2
|
|
1644
|
+
)
|
|
1645
|
+
|
|
1646
|
+
if refine is None:
|
|
1647
|
+
requested = _VALID_REFINE_LEVELS
|
|
1648
|
+
else:
|
|
1649
|
+
requested = refine
|
|
1650
|
+
|
|
1651
|
+
missing = [
|
|
1652
|
+
level for level in requested
|
|
1653
|
+
if level not in self._refine_levels
|
|
1654
|
+
]
|
|
1655
|
+
|
|
1656
|
+
if missing:
|
|
1657
|
+
print(
|
|
1658
|
+
f"Refinement level(s) {missing} were not included "
|
|
1659
|
+
f"at initialization. \n"
|
|
1660
|
+
f"To include these levels, re-initialize AzimuthNN "
|
|
1661
|
+
f"with refine={requested}."
|
|
1662
|
+
)
|
|
1663
|
+
return
|
|
1664
|
+
|
|
1665
|
+
pass
|
|
1666
|
+
|
|
1667
|
+
def azimuth_embed(self):
|
|
1668
|
+
"""
|
|
1669
|
+
Extract embeddings from the Azimuth model's embedding layer.
|
|
1670
|
+
|
|
1671
|
+
This method extracts cell embeddings from a pre-defined layer
|
|
1672
|
+
in the inference model and stores them in the embeddings
|
|
1673
|
+
dictionary under the key 'azimuth_embed', replacing the
|
|
1674
|
+
original model-specific key.
|
|
1675
|
+
|
|
1676
|
+
To extract embeddings from a different layer in the model, use
|
|
1677
|
+
AzimuthNN_base class for more fine grained control.
|
|
1678
|
+
|
|
1679
|
+
Returns
|
|
1680
|
+
-------
|
|
1681
|
+
numpy.ndarray
|
|
1682
|
+
The extracted embeddings, with shape
|
|
1683
|
+
(n_cells, embedding_dimension).
|
|
1684
|
+
|
|
1685
|
+
Raises
|
|
1686
|
+
------
|
|
1687
|
+
AssertionError
|
|
1688
|
+
If inference model hasn't been run yet.
|
|
1689
|
+
"""
|
|
1690
|
+
|
|
1691
|
+
azimuth_embedding_layer_name = self.model_meta[
|
|
1692
|
+
'inference_model_embedding_layer'
|
|
1693
|
+
]
|
|
1694
|
+
|
|
1695
|
+
azimuth_embeddings = self.inference_model_embeddings(
|
|
1696
|
+
embedding_layer_name = azimuth_embedding_layer_name
|
|
1697
|
+
)
|
|
1698
|
+
|
|
1699
|
+
self.embeddings['azimuth_embed'] = azimuth_embeddings
|
|
1700
|
+
del self.embeddings[
|
|
1701
|
+
f'{self.inference_model_name}_'
|
|
1702
|
+
f'{azimuth_embedding_layer_name}_embed'
|
|
1703
|
+
]
|
|
1704
|
+
|
|
1705
|
+
return azimuth_embeddings
|
|
1706
|
+
|
|
1707
|
+
def azimuth_umap(
|
|
1708
|
+
self,
|
|
1709
|
+
n_neighbors=30,
|
|
1710
|
+
n_components=2,
|
|
1711
|
+
metric='cosine',
|
|
1712
|
+
min_dist=0.3,
|
|
1713
|
+
umap_lr=1.0,
|
|
1714
|
+
umap_seed=42,
|
|
1715
|
+
spread=1.0,
|
|
1716
|
+
verbose=True,
|
|
1717
|
+
init='spectral'
|
|
1718
|
+
):
|
|
1719
|
+
"""
|
|
1720
|
+
Generate UMAP projection from Azimuth embeddings.
|
|
1721
|
+
|
|
1722
|
+
This method creates a UMAP projection from previously extracted
|
|
1723
|
+
Azimuth embeddings and stores it in the umaps dictionary.
|
|
1724
|
+
|
|
1725
|
+
Parameters
|
|
1726
|
+
----------
|
|
1727
|
+
n_neighbors : int, default=30
|
|
1728
|
+
Number of neighbors to consider for each point in UMAP.
|
|
1729
|
+
n_components : int, default=2
|
|
1730
|
+
Dimensionality of the UMAP projection.
|
|
1731
|
+
metric : str, default='cosine'
|
|
1732
|
+
Distance metric to use for UMAP.
|
|
1733
|
+
min_dist : float, default=0.3
|
|
1734
|
+
Minimum distance between points in the UMAP projection.
|
|
1735
|
+
umap_lr : float, default=1.0
|
|
1736
|
+
UMAP learning rate.
|
|
1737
|
+
umap_seed : int, default=42
|
|
1738
|
+
Random seed for UMAP for reproducibility.
|
|
1739
|
+
spread : float, default=1.0
|
|
1740
|
+
Scales the effective scale of embedded points.
|
|
1741
|
+
verbose : bool, default=True
|
|
1742
|
+
Whether to display progress during UMAP computation.
|
|
1743
|
+
init : str, default='spectral'
|
|
1744
|
+
Initialization method for UMAP.
|
|
1745
|
+
|
|
1746
|
+
Returns
|
|
1747
|
+
-------
|
|
1748
|
+
numpy.ndarray
|
|
1749
|
+
The UMAP projection, with shape (n_cells, n_components).
|
|
1750
|
+
|
|
1751
|
+
Raises
|
|
1752
|
+
------
|
|
1753
|
+
AssertionError
|
|
1754
|
+
If 'azimuth_embed' embeddings haven't been generated yet.
|
|
1755
|
+
"""
|
|
1756
|
+
|
|
1757
|
+
assert 'azimuth_embed' in self.embeddings.keys(), (
|
|
1758
|
+
"Extract azimuth_embed first before creating umap."
|
|
1759
|
+
)
|
|
1760
|
+
|
|
1761
|
+
umap_class = Umaps(
|
|
1762
|
+
n_neighbors=n_neighbors,
|
|
1763
|
+
n_components=n_components,
|
|
1764
|
+
metric=metric,
|
|
1765
|
+
min_dist=min_dist,
|
|
1766
|
+
umap_lr=umap_lr,
|
|
1767
|
+
umap_seed=umap_seed,
|
|
1768
|
+
spread=spread,
|
|
1769
|
+
verbose=verbose,
|
|
1770
|
+
init=init
|
|
1771
|
+
)
|
|
1772
|
+
|
|
1773
|
+
umap_gen = umap_class.create_umap(
|
|
1774
|
+
self.embeddings['azimuth_embed']
|
|
1775
|
+
)
|
|
1776
|
+
self.umaps['azimuth_umap'] = umap_gen
|
|
1777
|
+
|
|
1778
|
+
return umap_gen
|
|
1779
|
+
|
|
1780
|
+
def azimuth_embed_and_umap(
|
|
1781
|
+
self,
|
|
1782
|
+
n_neighbors=30,
|
|
1783
|
+
n_components=2,
|
|
1784
|
+
metric='cosine',
|
|
1785
|
+
min_dist=0.3,
|
|
1786
|
+
umap_lr=1.0,
|
|
1787
|
+
umap_seed=42,
|
|
1788
|
+
spread=1.0,
|
|
1789
|
+
verbose=True,
|
|
1790
|
+
init='spectral'
|
|
1791
|
+
):
|
|
1792
|
+
"""
|
|
1793
|
+
Extract embeddings and generate UMAP projection in one step.
|
|
1794
|
+
|
|
1795
|
+
This method provides a convenient wrapper that combines the
|
|
1796
|
+
functionality of azimuth_embed() and azimuth_umap() methods.
|
|
1797
|
+
It extracts embeddings from the inference model and immediately
|
|
1798
|
+
computes a UMAP projection, storing both results.
|
|
1799
|
+
|
|
1800
|
+
Parameters
|
|
1801
|
+
----------
|
|
1802
|
+
n_neighbors : int, default=30
|
|
1803
|
+
Number of neighbors to consider for each point in UMAP.
|
|
1804
|
+
n_components : int, default=2
|
|
1805
|
+
Dimensionality of the UMAP projection.
|
|
1806
|
+
metric : str, default='cosine'
|
|
1807
|
+
Distance metric to use for UMAP.
|
|
1808
|
+
min_dist : float, default=0.3
|
|
1809
|
+
Minimum distance between points in the UMAP projection.
|
|
1810
|
+
umap_lr : float, default=1.0
|
|
1811
|
+
UMAP learning rate.
|
|
1812
|
+
umap_seed : int, default=42
|
|
1813
|
+
Random seed for UMAP for reproducibility.
|
|
1814
|
+
spread : float, default=1.0
|
|
1815
|
+
Scales the effective scale of embedded points.
|
|
1816
|
+
verbose : bool, default=True
|
|
1817
|
+
Whether to display progress during UMAP computation.
|
|
1818
|
+
init : str, default='spectral'
|
|
1819
|
+
Initialization method for UMAP.
|
|
1820
|
+
|
|
1821
|
+
Returns
|
|
1822
|
+
-------
|
|
1823
|
+
tuple
|
|
1824
|
+
A tuple containing:
|
|
1825
|
+
- numpy.ndarray: The extracted embeddings
|
|
1826
|
+
- numpy.ndarray: The UMAP projection
|
|
1827
|
+
|
|
1828
|
+
Raises
|
|
1829
|
+
------
|
|
1830
|
+
AssertionError
|
|
1831
|
+
If inference model hasn't been run yet.
|
|
1832
|
+
"""
|
|
1833
|
+
|
|
1834
|
+
azimuth_embedding_layer_name = self.model_meta[
|
|
1835
|
+
'inference_model_embedding_layer'
|
|
1836
|
+
]
|
|
1837
|
+
|
|
1838
|
+
(
|
|
1839
|
+
azimuth_embeddings,
|
|
1840
|
+
azimuth_umap
|
|
1841
|
+
) = self.inference_embeddings_and_umap(
|
|
1842
|
+
embedding_layer_name = azimuth_embedding_layer_name,
|
|
1843
|
+
n_neighbors=n_neighbors,
|
|
1844
|
+
n_components=n_components,
|
|
1845
|
+
metric=metric,
|
|
1846
|
+
min_dist=min_dist,
|
|
1847
|
+
umap_lr=umap_lr,
|
|
1848
|
+
umap_seed=umap_seed,
|
|
1849
|
+
spread=spread,
|
|
1850
|
+
verbose=verbose,
|
|
1851
|
+
init=init
|
|
1852
|
+
)
|
|
1853
|
+
|
|
1854
|
+
embed_key_og = (
|
|
1855
|
+
f'{self.inference_model_name}_'
|
|
1856
|
+
f'{azimuth_embedding_layer_name}_embed'
|
|
1857
|
+
)
|
|
1858
|
+
umap_key_og = (
|
|
1859
|
+
f'{self.inference_model_name}_'
|
|
1860
|
+
f'{azimuth_embedding_layer_name}_umap'
|
|
1861
|
+
)
|
|
1862
|
+
|
|
1863
|
+
self.embeddings['azimuth_embed'] = azimuth_embeddings
|
|
1864
|
+
self.umaps['azimuth_umap'] = azimuth_umap
|
|
1865
|
+
|
|
1866
|
+
del self.embeddings[embed_key_og]
|
|
1867
|
+
del self.umaps[umap_key_og]
|
|
1868
|
+
|
|
1869
|
+
return azimuth_embeddings, azimuth_umap
|
|
1870
|
+
|
|
1871
|
+
|
|
1872
|
+
|
|
1873
|
+
|
|
1874
|
+
################################################################################
|
|
1875
|
+
########### functions for python and R script ##################################
|
|
1876
|
+
################################################################################
|
|
1877
|
+
################################################################################
|
|
1878
|
+
|
|
1879
|
+
|
|
1880
|
+
|
|
1881
|
+
|
|
1882
|
+
|
|
1883
|
+
########### annotate_core, core of python/R scripts ############################
|
|
1884
|
+
|
|
1885
|
+
|
|
1886
|
+
|
|
1887
|
+
def annotate_core(
|
|
1888
|
+
X_query,
|
|
1889
|
+
query_features,
|
|
1890
|
+
cells_meta,
|
|
1891
|
+
annotation_pipeline,
|
|
1892
|
+
eval_batch_size,
|
|
1893
|
+
normalization_override,
|
|
1894
|
+
norm_check_batch_size,
|
|
1895
|
+
output_mode,
|
|
1896
|
+
refine_labels,
|
|
1897
|
+
map_to_cl,
|
|
1898
|
+
include_cl_id,
|
|
1899
|
+
extract_embeddings,
|
|
1900
|
+
umap_embeddings,
|
|
1901
|
+
n_neighbors,
|
|
1902
|
+
n_components,
|
|
1903
|
+
metric,
|
|
1904
|
+
min_dist,
|
|
1905
|
+
umap_lr,
|
|
1906
|
+
umap_seed,
|
|
1907
|
+
spread,
|
|
1908
|
+
verbose,
|
|
1909
|
+
init,
|
|
1910
|
+
model_version=model_version_default
|
|
1911
|
+
# adding a default here, so the R script does not need mods
|
|
1912
|
+
# to access the default.
|
|
1913
|
+
):
|
|
1914
|
+
"""
|
|
1915
|
+
Core function for cell type annotation using the Azimuth neural
|
|
1916
|
+
network, designed primarily for script-based usage.
|
|
1917
|
+
|
|
1918
|
+
While AzimuthNN and AzimuthNN_base classes provide interactive
|
|
1919
|
+
functionality for exploratory analysis, this function offers a
|
|
1920
|
+
one-step method for automated annotation via Python or R scripts. It
|
|
1921
|
+
performs the complete annotation workflow in a single function
|
|
1922
|
+
call: data preprocessing, model inference, confidence calibration,
|
|
1923
|
+
label generation, optional label refinement, optional Cell Ontology
|
|
1924
|
+
mapping, and optional embedding/UMAP generation.
|
|
1925
|
+
|
|
1926
|
+
As of v0.3.0, this function uses the AzimuthNN class internally,
|
|
1927
|
+
which performs inference, calibration, and refinement in minibatches
|
|
1928
|
+
for improved memory efficiency.
|
|
1929
|
+
|
|
1930
|
+
Parameters
|
|
1931
|
+
----------
|
|
1932
|
+
X_query : scipy.sparse.csr_matrix
|
|
1933
|
+
Expression matrix with cells as rows and genes as columns.
|
|
1934
|
+
query_features : list of str
|
|
1935
|
+
List of feature names (gene identifiers) corresponding to
|
|
1936
|
+
columns in X_query.
|
|
1937
|
+
cells_meta : pandas.DataFrame
|
|
1938
|
+
Metadata for cells, with rows corresponding to cells in X_query.
|
|
1939
|
+
annotation_pipeline : str
|
|
1940
|
+
Type of annotation pipeline to use for cell type prediction.
|
|
1941
|
+
eval_batch_size : int
|
|
1942
|
+
Batch size to use during model inference.
|
|
1943
|
+
normalization_override : bool
|
|
1944
|
+
If True, skips normalization check and forces processing to
|
|
1945
|
+
continue.
|
|
1946
|
+
norm_check_batch_size : int
|
|
1947
|
+
Number of cells to sample for normalization check.
|
|
1948
|
+
output_mode : str
|
|
1949
|
+
Controls the verbosity of output in the cell meta dataframe.
|
|
1950
|
+
Options are 'minimal' or 'detailed'.
|
|
1951
|
+
refine_labels : bool
|
|
1952
|
+
Whether to perform label refinement at broad, medium, and fine
|
|
1953
|
+
levels.
|
|
1954
|
+
map_to_cl : list of str or None
|
|
1955
|
+
List of column names in cells_meta to map to Cell Ontology
|
|
1956
|
+
terms after annotation is complete. Each named column must
|
|
1957
|
+
exist in cells_meta at the time of mapping, so columns produced
|
|
1958
|
+
by the annotation pipeline (e.g. 'azimuth_broad', 'azimuth_fine')
|
|
1959
|
+
are valid targets. If None, no mapping is applied.
|
|
1960
|
+
include_cl_id : bool
|
|
1961
|
+
If True, also adds a CL identifier column (e.g. 'CL:0000236')
|
|
1962
|
+
alongside each CL label column produced by map_to_cl. Has no
|
|
1963
|
+
effect if map_to_cl is None.
|
|
1964
|
+
extract_embeddings : bool
|
|
1965
|
+
Whether to extract embeddings from the model.
|
|
1966
|
+
umap_embeddings : bool
|
|
1967
|
+
Whether to generate UMAP projections from the embeddings.
|
|
1968
|
+
Requires extract_embeddings=True.
|
|
1969
|
+
n_neighbors : int
|
|
1970
|
+
Number of neighbors to consider for each point in UMAP.
|
|
1971
|
+
n_components : int
|
|
1972
|
+
Dimensionality of the UMAP projection.
|
|
1973
|
+
metric : str
|
|
1974
|
+
Distance metric to use for UMAP.
|
|
1975
|
+
min_dist : float
|
|
1976
|
+
Minimum distance between points in the UMAP projection.
|
|
1977
|
+
umap_lr : float
|
|
1978
|
+
UMAP learning rate.
|
|
1979
|
+
umap_seed : int
|
|
1980
|
+
Random seed for UMAP for reproducibility.
|
|
1981
|
+
spread : float
|
|
1982
|
+
Scales the effective scale of embedded points.
|
|
1983
|
+
verbose : bool
|
|
1984
|
+
Whether to display progress during UMAP computation.
|
|
1985
|
+
init : str
|
|
1986
|
+
Initialization method for UMAP.
|
|
1987
|
+
model_version : str
|
|
1988
|
+
Model version to use, e.g. 'v0' or 'v1'. Defaults to
|
|
1989
|
+
model_version_default as defined in this module.
|
|
1990
|
+
|
|
1991
|
+
Returns
|
|
1992
|
+
-------
|
|
1993
|
+
dict
|
|
1994
|
+
A dictionary containing:
|
|
1995
|
+
- 'azimuth_object': The instantiated AzimuthNN object
|
|
1996
|
+
- 'embeddings_dict': Dictionary of computed embeddings
|
|
1997
|
+
- 'umap_dict': Dictionary of computed UMAP projections
|
|
1998
|
+
- 'cells_meta': Updated cell metadata with annotations and,
|
|
1999
|
+
if map_to_cl was specified, Cell Ontology columns
|
|
2000
|
+
|
|
2001
|
+
Raises
|
|
2002
|
+
------
|
|
2003
|
+
TypeError
|
|
2004
|
+
If normalization_override, extract_embeddings, umap_embeddings,
|
|
2005
|
+
refine_labels, or include_cl_id are not boolean values, if
|
|
2006
|
+
norm_check_batch_size is not an integer, or if map_to_cl is
|
|
2007
|
+
not a list of strings or None.
|
|
2008
|
+
ValueError
|
|
2009
|
+
If output_mode is not 'minimal' or 'detailed', or if
|
|
2010
|
+
umap_embeddings is True but extract_embeddings is False.
|
|
2011
|
+
|
|
2012
|
+
Notes
|
|
2013
|
+
-----
|
|
2014
|
+
This function is designed to be the core engine for script-based
|
|
2015
|
+
automated annotation workflows. Unlike the interactive AzimuthNN and
|
|
2016
|
+
AzimuthNN_base classes which allow step-by-step exploration and
|
|
2017
|
+
visualization, this function executes the entire annotation pipeline
|
|
2018
|
+
in one call.
|
|
2019
|
+
|
|
2020
|
+
It's particularly useful for:
|
|
2021
|
+
- Batch processing of multiple datasets
|
|
2022
|
+
- Integration into automated analysis pipelines
|
|
2023
|
+
- Creating wrappers for other languages (like R)
|
|
2024
|
+
|
|
2025
|
+
Cell Ontology mapping (map_to_cl) is applied after the full
|
|
2026
|
+
annotation pipeline completes, so columns added by the pipeline
|
|
2027
|
+
(e.g. 'azimuth_broad', 'azimuth_fine') can be specified directly.
|
|
2028
|
+
|
|
2029
|
+
Examples
|
|
2030
|
+
--------
|
|
2031
|
+
>>> from scipy.sparse import csr_matrix
|
|
2032
|
+
>>> import pandas as pd
|
|
2033
|
+
>>> import numpy as np
|
|
2034
|
+
>>>
|
|
2035
|
+
>>> X = csr_matrix(np.random.rand(100, 1000))
|
|
2036
|
+
>>> features = [f"gene_{i}" for i in range(1000)]
|
|
2037
|
+
>>> meta = pd.DataFrame(index=range(100))
|
|
2038
|
+
>>>
|
|
2039
|
+
>>> # Run annotation with Cell Ontology mapping
|
|
2040
|
+
>>> results = annotate_core(
|
|
2041
|
+
... X, features, meta,
|
|
2042
|
+
... annotation_pipeline='supervised',
|
|
2043
|
+
... eval_batch_size=8192,
|
|
2044
|
+
... normalization_override=False,
|
|
2045
|
+
... norm_check_batch_size=100,
|
|
2046
|
+
... output_mode='minimal',
|
|
2047
|
+
... refine_labels=True,
|
|
2048
|
+
... map_to_cl=['azimuth_broad', 'azimuth_fine'],
|
|
2049
|
+
... include_cl_id=True,
|
|
2050
|
+
... extract_embeddings=True,
|
|
2051
|
+
... umap_embeddings=True,
|
|
2052
|
+
... n_neighbors=30,
|
|
2053
|
+
... n_components=2,
|
|
2054
|
+
... metric='cosine',
|
|
2055
|
+
... min_dist=0.3,
|
|
2056
|
+
... umap_lr=1.0,
|
|
2057
|
+
... umap_seed=42,
|
|
2058
|
+
... spread=1.0,
|
|
2059
|
+
... verbose=True,
|
|
2060
|
+
... init='spectral'
|
|
2061
|
+
... )
|
|
2062
|
+
>>> annotated_meta = results['cells_meta']
|
|
2063
|
+
>>> embeddings = results['embeddings_dict']['azimuth_embed']
|
|
2064
|
+
>>> umap_coords = results['umap_dict']['azimuth_umap']
|
|
2065
|
+
"""
|
|
2066
|
+
|
|
2067
|
+
if not isinstance(normalization_override, bool):
|
|
2068
|
+
raise TypeError("normalization override must be a bool")
|
|
2069
|
+
|
|
2070
|
+
if not isinstance(norm_check_batch_size, int):
|
|
2071
|
+
raise TypeError("norm_check_batch_size must be an integer")
|
|
2072
|
+
|
|
2073
|
+
if output_mode not in ['minimal','detailed']:
|
|
2074
|
+
raise ValueError(
|
|
2075
|
+
"mode for output processing should be either "
|
|
2076
|
+
"'minimal' or 'detailed'"
|
|
2077
|
+
)
|
|
2078
|
+
|
|
2079
|
+
if not isinstance(extract_embeddings, bool):
|
|
2080
|
+
raise TypeError("extract_embeddings argument should be boolean")
|
|
2081
|
+
|
|
2082
|
+
if not isinstance(umap_embeddings, bool):
|
|
2083
|
+
raise TypeError("umap_embeddings argument should be boolean")
|
|
2084
|
+
|
|
2085
|
+
if not isinstance(refine_labels, bool):
|
|
2086
|
+
raise TypeError("refine_labels argument should be boolean")
|
|
2087
|
+
|
|
2088
|
+
if map_to_cl is not None:
|
|
2089
|
+
if not isinstance(map_to_cl, list):
|
|
2090
|
+
raise TypeError(
|
|
2091
|
+
"map_to_cl must be a list of column name strings or None."
|
|
2092
|
+
)
|
|
2093
|
+
if not all(isinstance(col, str) for col in map_to_cl):
|
|
2094
|
+
raise TypeError(
|
|
2095
|
+
"All entries in map_to_cl must be strings."
|
|
2096
|
+
)
|
|
2097
|
+
|
|
2098
|
+
if not isinstance(include_cl_id, bool):
|
|
2099
|
+
raise TypeError("include_cl_id must be a bool.")
|
|
2100
|
+
|
|
2101
|
+
if umap_embeddings:
|
|
2102
|
+
if not extract_embeddings:
|
|
2103
|
+
raise ValueError(
|
|
2104
|
+
"Embeddings must be extracted to create umap.\n"
|
|
2105
|
+
"Set extract_embeddings to True."
|
|
2106
|
+
)
|
|
2107
|
+
|
|
2108
|
+
# load model meta to print model name before pipeline runs
|
|
2109
|
+
_version_module = importlib.import_module(
|
|
2110
|
+
f"panhumanpy._tools.{model_version}"
|
|
2111
|
+
)
|
|
2112
|
+
_model_meta = _version_module.model_meta
|
|
2113
|
+
|
|
2114
|
+
print("Reference model and parameters:")
|
|
2115
|
+
print(f" Model version: {model_version}")
|
|
2116
|
+
print(f" Model name: {_model_meta['inference_model_name']}")
|
|
2117
|
+
print(f" Evaluation batch size: {eval_batch_size}")
|
|
2118
|
+
print(f" Extract embeddings: {extract_embeddings}")
|
|
2119
|
+
print(f" Run umap: {umap_embeddings}")
|
|
2120
|
+
print(f" Refine labels in postprocessing: {refine_labels}")
|
|
2121
|
+
print(f" Map to Cell Ontology columns: {map_to_cl}")
|
|
2122
|
+
print(f" Include CL ID: {include_cl_id}")
|
|
2123
|
+
|
|
2124
|
+
# construct a minimal AnnData from pre-extracted components.
|
|
2125
|
+
# this wraps references, no data is copied.
|
|
2126
|
+
query_adata = anndata.AnnData(
|
|
2127
|
+
X = X_query,
|
|
2128
|
+
obs = cells_meta,
|
|
2129
|
+
var = pd.DataFrame(index=query_features)
|
|
2130
|
+
)
|
|
2131
|
+
|
|
2132
|
+
azimuth = AzimuthNN(
|
|
2133
|
+
query_adata,
|
|
2134
|
+
annotation_pipeline = annotation_pipeline,
|
|
2135
|
+
model_version = model_version,
|
|
2136
|
+
eval_batch_size = eval_batch_size,
|
|
2137
|
+
normalization_override = normalization_override,
|
|
2138
|
+
norm_check_batch_size = norm_check_batch_size,
|
|
2139
|
+
output_mode = output_mode,
|
|
2140
|
+
refine = refine_labels
|
|
2141
|
+
)
|
|
2142
|
+
|
|
2143
|
+
if extract_embeddings:
|
|
2144
|
+
if umap_embeddings:
|
|
2145
|
+
azimuth.azimuth_embed_and_umap(
|
|
2146
|
+
n_neighbors=n_neighbors,
|
|
2147
|
+
n_components=n_components,
|
|
2148
|
+
metric=metric,
|
|
2149
|
+
min_dist=min_dist,
|
|
2150
|
+
umap_lr=umap_lr,
|
|
2151
|
+
umap_seed=umap_seed,
|
|
2152
|
+
spread=spread,
|
|
2153
|
+
verbose=verbose,
|
|
2154
|
+
init=init
|
|
2155
|
+
)
|
|
2156
|
+
else:
|
|
2157
|
+
azimuth.azimuth_embed()
|
|
2158
|
+
|
|
2159
|
+
if map_to_cl is not None:
|
|
2160
|
+
for col in map_to_cl:
|
|
2161
|
+
azimuth.map_to_cell_ontology(
|
|
2162
|
+
src_col=col,
|
|
2163
|
+
include_cl_id=include_cl_id
|
|
2164
|
+
)
|
|
2165
|
+
|
|
2166
|
+
core_outputs = {
|
|
2167
|
+
'azimuth_object' : azimuth,
|
|
2168
|
+
'embeddings_dict' : azimuth.embeddings,
|
|
2169
|
+
'umap_dict' : azimuth.umaps,
|
|
2170
|
+
'cells_meta' : azimuth.cells_meta
|
|
2171
|
+
}
|
|
2172
|
+
|
|
2173
|
+
return core_outputs
|
|
2174
|
+
|
|
2175
|
+
|
|
2176
|
+
|
|
2177
|
+
|
|
2178
|
+
|
|
2179
|
+
############################ arg parsing ###############################
|
|
2180
|
+
|
|
2181
|
+
def arg_parse_in():
|
|
2182
|
+
"""
|
|
2183
|
+
Parse command line arguments for the Azimuth cell annotation tool.
|
|
2184
|
+
|
|
2185
|
+
Sets up argument parser with all parameters required for the annotation
|
|
2186
|
+
pipeline, including input file handling, model configuration, and
|
|
2187
|
+
visualization options.
|
|
2188
|
+
|
|
2189
|
+
Returns
|
|
2190
|
+
-------
|
|
2191
|
+
argparse.Namespace
|
|
2192
|
+
Parsed command line arguments
|
|
2193
|
+
"""
|
|
2194
|
+
|
|
2195
|
+
print("Parsing arguments... \n")
|
|
2196
|
+
print("\n")
|
|
2197
|
+
|
|
2198
|
+
|
|
2199
|
+
parser = argparse.ArgumentParser()
|
|
2200
|
+
parser.add_argument(
|
|
2201
|
+
"filepath",
|
|
2202
|
+
help=(
|
|
2203
|
+
"enter abs file path to the query."
|
|
2204
|
+
" Query should be in h5ad format."
|
|
2205
|
+
),
|
|
2206
|
+
type=str
|
|
2207
|
+
)
|
|
2208
|
+
|
|
2209
|
+
|
|
2210
|
+
parser.add_argument(
|
|
2211
|
+
"-fn",
|
|
2212
|
+
"--feature_names_col",
|
|
2213
|
+
default=None,
|
|
2214
|
+
help=(
|
|
2215
|
+
"enter the column name where the "
|
|
2216
|
+
"feature names are stored in query.var"
|
|
2217
|
+
" where query is the anndata object read "
|
|
2218
|
+
"from the h5ad."
|
|
2219
|
+
),
|
|
2220
|
+
type=str
|
|
2221
|
+
)
|
|
2222
|
+
|
|
2223
|
+
parser.add_argument(
|
|
2224
|
+
"-ap",
|
|
2225
|
+
"--annotation_pipeline",
|
|
2226
|
+
default='supervised',
|
|
2227
|
+
help=(
|
|
2228
|
+
"enter annotation pipeline"
|
|
2229
|
+
),
|
|
2230
|
+
type=str
|
|
2231
|
+
)
|
|
2232
|
+
|
|
2233
|
+
parser.add_argument(
|
|
2234
|
+
"-mv",
|
|
2235
|
+
"--model_version",
|
|
2236
|
+
default=model_version_default,
|
|
2237
|
+
help=(
|
|
2238
|
+
"enter model version to use"
|
|
2239
|
+
),
|
|
2240
|
+
type=str
|
|
2241
|
+
)
|
|
2242
|
+
|
|
2243
|
+
parser.add_argument(
|
|
2244
|
+
"-ebs",
|
|
2245
|
+
"--eval_batch_size",
|
|
2246
|
+
default=8192,
|
|
2247
|
+
help=(
|
|
2248
|
+
"enter the evaluation batch size suitable to "
|
|
2249
|
+
"your hardware, defaults to 8192"
|
|
2250
|
+
),
|
|
2251
|
+
type=int
|
|
2252
|
+
)
|
|
2253
|
+
|
|
2254
|
+
parser.add_argument(
|
|
2255
|
+
"-norm",
|
|
2256
|
+
"--normalization_override",
|
|
2257
|
+
action = "store_true",
|
|
2258
|
+
help="Override normalisation."
|
|
2259
|
+
)
|
|
2260
|
+
|
|
2261
|
+
parser.add_argument(
|
|
2262
|
+
"-ncbs",
|
|
2263
|
+
"--norm_check_batch_size",
|
|
2264
|
+
default=100,
|
|
2265
|
+
help=(
|
|
2266
|
+
"enter the number of cells over which "
|
|
2267
|
+
"normalization will be verified, defaults "
|
|
2268
|
+
"to 100"
|
|
2269
|
+
),
|
|
2270
|
+
type=int
|
|
2271
|
+
)
|
|
2272
|
+
|
|
2273
|
+
parser.add_argument(
|
|
2274
|
+
"-om",
|
|
2275
|
+
"--output_mode",
|
|
2276
|
+
default='minimal',
|
|
2277
|
+
help=(
|
|
2278
|
+
"enter output mode, must be either 'minimal'"
|
|
2279
|
+
" or 'detailed'."
|
|
2280
|
+
),
|
|
2281
|
+
type=str
|
|
2282
|
+
)
|
|
2283
|
+
|
|
2284
|
+
parser.add_argument(
|
|
2285
|
+
"-rf",
|
|
2286
|
+
"--refine_labels",
|
|
2287
|
+
action = "store_false",
|
|
2288
|
+
help="Skip label refinement."
|
|
2289
|
+
)
|
|
2290
|
+
|
|
2291
|
+
parser.add_argument(
|
|
2292
|
+
"-mcl",
|
|
2293
|
+
"--map_to_cl",
|
|
2294
|
+
default=None,
|
|
2295
|
+
nargs='+',
|
|
2296
|
+
help=(
|
|
2297
|
+
"One or more column names in the cell "
|
|
2298
|
+
"metadata to map to Cell Ontology labels. "
|
|
2299
|
+
"e.g. -mcl azimuth_broad azimuth_fine"
|
|
2300
|
+
),
|
|
2301
|
+
type=str
|
|
2302
|
+
)
|
|
2303
|
+
|
|
2304
|
+
parser.add_argument(
|
|
2305
|
+
"-clid",
|
|
2306
|
+
"--include_cl_id",
|
|
2307
|
+
action="store_true",
|
|
2308
|
+
help=(
|
|
2309
|
+
"If set, also adds CL ID columns "
|
|
2310
|
+
"(e.g. CL:0000236) alongside CL label "
|
|
2311
|
+
"columns. Default: False."
|
|
2312
|
+
)
|
|
2313
|
+
)
|
|
2314
|
+
|
|
2315
|
+
parser.add_argument(
|
|
2316
|
+
"-em",
|
|
2317
|
+
"--extract_embeddings",
|
|
2318
|
+
action = "store_false",
|
|
2319
|
+
help="Skip embedding extraction"
|
|
2320
|
+
)
|
|
2321
|
+
|
|
2322
|
+
|
|
2323
|
+
parser.add_argument(
|
|
2324
|
+
"-umap",
|
|
2325
|
+
"--umap_embeddings",
|
|
2326
|
+
action = "store_false",
|
|
2327
|
+
help="Skip UMAP creation"
|
|
2328
|
+
)
|
|
2329
|
+
|
|
2330
|
+
|
|
2331
|
+
|
|
2332
|
+
parser.add_argument(
|
|
2333
|
+
"-nnbrs",
|
|
2334
|
+
"--n_neighbors",
|
|
2335
|
+
default=30,
|
|
2336
|
+
help=(
|
|
2337
|
+
"n_neighbors param for umaps, defaults "
|
|
2338
|
+
"to Seurat default 30"
|
|
2339
|
+
),
|
|
2340
|
+
type=int
|
|
2341
|
+
)
|
|
2342
|
+
|
|
2343
|
+
parser.add_argument(
|
|
2344
|
+
"-nc",
|
|
2345
|
+
"--n_components",
|
|
2346
|
+
default=2,
|
|
2347
|
+
help=(
|
|
2348
|
+
"n_components param for umaps, defaults "
|
|
2349
|
+
"to Seurat default 2"
|
|
2350
|
+
),
|
|
2351
|
+
type=int
|
|
2352
|
+
)
|
|
2353
|
+
|
|
2354
|
+
parser.add_argument(
|
|
2355
|
+
"-me",
|
|
2356
|
+
"--metric",
|
|
2357
|
+
default='cosine',
|
|
2358
|
+
help=(
|
|
2359
|
+
"metric param for umaps, defaults to "
|
|
2360
|
+
"Seurat default 'cosine'"
|
|
2361
|
+
),
|
|
2362
|
+
type=str
|
|
2363
|
+
)
|
|
2364
|
+
|
|
2365
|
+
parser.add_argument(
|
|
2366
|
+
"-mdt",
|
|
2367
|
+
"--min_dist",
|
|
2368
|
+
default=0.3,
|
|
2369
|
+
help=(
|
|
2370
|
+
"min_dist param for umaps, defaults to "
|
|
2371
|
+
"Seurat default 0.3"
|
|
2372
|
+
),
|
|
2373
|
+
type=float
|
|
2374
|
+
)
|
|
2375
|
+
|
|
2376
|
+
parser.add_argument(
|
|
2377
|
+
"-ulr",
|
|
2378
|
+
"--umap_lr",
|
|
2379
|
+
default=1.0,
|
|
2380
|
+
help=("learning_rate param for umaps, defaults "
|
|
2381
|
+
"to Seurat default 1.0"
|
|
2382
|
+
),
|
|
2383
|
+
type=float
|
|
2384
|
+
)
|
|
2385
|
+
|
|
2386
|
+
parser.add_argument(
|
|
2387
|
+
"-useed",
|
|
2388
|
+
"--umap_seed",
|
|
2389
|
+
default=42,
|
|
2390
|
+
help=(
|
|
2391
|
+
"random_state param for reproducibility of "
|
|
2392
|
+
"umaps, defaults to Seurat default 42"
|
|
2393
|
+
),
|
|
2394
|
+
type=int
|
|
2395
|
+
)
|
|
2396
|
+
|
|
2397
|
+
parser.add_argument(
|
|
2398
|
+
"-sp",
|
|
2399
|
+
"--spread",
|
|
2400
|
+
default=1.0,
|
|
2401
|
+
help=(
|
|
2402
|
+
"spread param for umaps, defaults to "
|
|
2403
|
+
"Seurat default 1.0"
|
|
2404
|
+
),
|
|
2405
|
+
type=float
|
|
2406
|
+
)
|
|
2407
|
+
|
|
2408
|
+
parser.add_argument(
|
|
2409
|
+
"-uv",
|
|
2410
|
+
"--umap_verbose",
|
|
2411
|
+
action="store_false",
|
|
2412
|
+
help="Hide UMAP progress"
|
|
2413
|
+
)
|
|
2414
|
+
|
|
2415
|
+
parser.add_argument(
|
|
2416
|
+
"-uin",
|
|
2417
|
+
"--umap_init",
|
|
2418
|
+
default="spectral",
|
|
2419
|
+
help=(
|
|
2420
|
+
"init param for umaps, defaults to "
|
|
2421
|
+
"'spectral', the other option is 'random'"
|
|
2422
|
+
),
|
|
2423
|
+
type=str
|
|
2424
|
+
)
|
|
2425
|
+
|
|
2426
|
+
parser.set_defaults(
|
|
2427
|
+
normalization_override=False,
|
|
2428
|
+
refine_labels=True,
|
|
2429
|
+
include_cl_id=False,
|
|
2430
|
+
extract_embeddings=True,
|
|
2431
|
+
umap_embeddings=True,
|
|
2432
|
+
umap_verbose=True
|
|
2433
|
+
)
|
|
2434
|
+
|
|
2435
|
+
|
|
2436
|
+
args = parser.parse_args()
|
|
2437
|
+
|
|
2438
|
+
return args
|
|
2439
|
+
|
|
2440
|
+
|
|
2441
|
+
|
|
2442
|
+
|
|
2443
|
+
def arg_parse_out(args):
|
|
2444
|
+
"""
|
|
2445
|
+
Convert parsed arguments to a dictionary for the annotation pipeline.
|
|
2446
|
+
|
|
2447
|
+
Takes the parsed command line arguments and transforms them into a
|
|
2448
|
+
structured dictionary that can be passed to the annotation functions.
|
|
2449
|
+
|
|
2450
|
+
Parameters
|
|
2451
|
+
----------
|
|
2452
|
+
args : argparse.Namespace
|
|
2453
|
+
Parsed command line arguments from arg_parse_in()
|
|
2454
|
+
|
|
2455
|
+
Returns
|
|
2456
|
+
-------
|
|
2457
|
+
dict
|
|
2458
|
+
Dictionary of arguments ready for use in annotation functions
|
|
2459
|
+
"""
|
|
2460
|
+
print("Reading arguments... \n")
|
|
2461
|
+
print("\n")
|
|
2462
|
+
|
|
2463
|
+
|
|
2464
|
+
|
|
2465
|
+
query_filepath = args.filepath
|
|
2466
|
+
feature_names_col = args.feature_names_col
|
|
2467
|
+
annotation_pipeline = args.annotation_pipeline
|
|
2468
|
+
model_version = args.model_version
|
|
2469
|
+
eval_batch_size = args.eval_batch_size
|
|
2470
|
+
normalization_override = args.normalization_override
|
|
2471
|
+
norm_check_batch_size = args.norm_check_batch_size
|
|
2472
|
+
output_mode = args.output_mode
|
|
2473
|
+
refine_labels = args.refine_labels
|
|
2474
|
+
map_to_cl = args.map_to_cl
|
|
2475
|
+
include_cl_id = args.include_cl_id
|
|
2476
|
+
extract_embeddings = args.extract_embeddings
|
|
2477
|
+
umap_embeddings = args.umap_embeddings
|
|
2478
|
+
n_neighbors = args.n_neighbors
|
|
2479
|
+
n_components = args.n_components
|
|
2480
|
+
metric = args.metric
|
|
2481
|
+
min_dist = args.min_dist
|
|
2482
|
+
umap_lr = args.umap_lr
|
|
2483
|
+
umap_seed = args.umap_seed
|
|
2484
|
+
spread = args.spread
|
|
2485
|
+
umap_verbose = args.umap_verbose
|
|
2486
|
+
umap_init = args.umap_init
|
|
2487
|
+
|
|
2488
|
+
arguments={
|
|
2489
|
+
'query_filepath' : query_filepath,
|
|
2490
|
+
'feature_names_col' : feature_names_col,
|
|
2491
|
+
'annotation_pipeline' : annotation_pipeline,
|
|
2492
|
+
'model_version': model_version,
|
|
2493
|
+
'eval_batch_size' : eval_batch_size,
|
|
2494
|
+
'normalization_override' : normalization_override,
|
|
2495
|
+
'norm_check_batch_size' : norm_check_batch_size,
|
|
2496
|
+
'output_mode' : output_mode,
|
|
2497
|
+
'refine_labels' : refine_labels,
|
|
2498
|
+
'map_to_cl' : map_to_cl,
|
|
2499
|
+
'include_cl_id' : include_cl_id,
|
|
2500
|
+
'extract_embeddings' : extract_embeddings,
|
|
2501
|
+
'umap_embeddings' : umap_embeddings,
|
|
2502
|
+
'n_neighbors' : n_neighbors,
|
|
2503
|
+
'n_components' : n_components,
|
|
2504
|
+
'metric' : metric,
|
|
2505
|
+
'min_dist' : min_dist,
|
|
2506
|
+
'umap_lr' : umap_lr,
|
|
2507
|
+
'umap_seed' : umap_seed,
|
|
2508
|
+
'spread' : spread,
|
|
2509
|
+
'verbose' : umap_verbose,
|
|
2510
|
+
'init' : umap_init
|
|
2511
|
+
}
|
|
2512
|
+
|
|
2513
|
+
return arguments
|
|
2514
|
+
|
|
2515
|
+
|
|
2516
|
+
############## annotate, executable python function ####################
|
|
2517
|
+
|
|
2518
|
+
|
|
2519
|
+
def annotate():
|
|
2520
|
+
"""
|
|
2521
|
+
Main entry point for command-line execution of the Azimuth cell
|
|
2522
|
+
annotation pipeline.
|
|
2523
|
+
|
|
2524
|
+
Parses command line arguments, loads the specified h5ad file, runs
|
|
2525
|
+
the annotation pipeline including confidence calibration via
|
|
2526
|
+
annotate_core(), and saves the results as a new h5ad file in the
|
|
2527
|
+
same directory as the input file with '_ANN' appended to the filename.
|
|
2528
|
+
|
|
2529
|
+
This function is intended to be called when the module is executed
|
|
2530
|
+
directly as a script and provides a complete workflow from argument
|
|
2531
|
+
parsing to saving results.
|
|
2532
|
+
|
|
2533
|
+
No parameters or return values as this function is designed to be
|
|
2534
|
+
the executable entry point for command-line usage.
|
|
2535
|
+
"""
|
|
2536
|
+
|
|
2537
|
+
args = arg_parse_in()
|
|
2538
|
+
arguments = arg_parse_out(args)
|
|
2539
|
+
|
|
2540
|
+
for key, value in arguments.items():
|
|
2541
|
+
globals()[key] = value
|
|
2542
|
+
|
|
2543
|
+
query_obj = ReadQueryObj(query_filepath)
|
|
2544
|
+
X_query = query_obj.X_query()
|
|
2545
|
+
query_features = query_obj.query_features(
|
|
2546
|
+
feature_names_col=feature_names_col
|
|
2547
|
+
)
|
|
2548
|
+
features_meta = query_obj.features_meta()
|
|
2549
|
+
cells_meta = query_obj.cells_meta()
|
|
2550
|
+
|
|
2551
|
+
core_outputs = annotate_core(
|
|
2552
|
+
X_query,
|
|
2553
|
+
query_features,
|
|
2554
|
+
cells_meta,
|
|
2555
|
+
annotation_pipeline,
|
|
2556
|
+
eval_batch_size,
|
|
2557
|
+
normalization_override,
|
|
2558
|
+
norm_check_batch_size,
|
|
2559
|
+
output_mode,
|
|
2560
|
+
refine_labels,
|
|
2561
|
+
map_to_cl,
|
|
2562
|
+
include_cl_id,
|
|
2563
|
+
extract_embeddings,
|
|
2564
|
+
umap_embeddings,
|
|
2565
|
+
n_neighbors,
|
|
2566
|
+
n_components,
|
|
2567
|
+
metric,
|
|
2568
|
+
min_dist,
|
|
2569
|
+
umap_lr,
|
|
2570
|
+
umap_seed,
|
|
2571
|
+
spread,
|
|
2572
|
+
verbose,
|
|
2573
|
+
init,
|
|
2574
|
+
model_version = model_version
|
|
2575
|
+
)
|
|
2576
|
+
|
|
2577
|
+
azimuth_object = core_outputs['azimuth_object']
|
|
2578
|
+
azimuth_object.features_meta = features_meta
|
|
2579
|
+
|
|
2580
|
+
dir_path = os.path.dirname(query_filepath)
|
|
2581
|
+
filename = os.path.basename(query_filepath)
|
|
2582
|
+
filename, ext = os.path.splitext(filename)
|
|
2583
|
+
out_path = os.path.join(dir_path, f'{filename}_ANN{ext}')
|
|
2584
|
+
|
|
2585
|
+
azimuth_object.pack_adata(save_path=out_path)
|
|
2586
|
+
|
|
2587
|
+
|
|
2588
|
+
####################################################################
|
|
2589
|
+
|
|
2590
|
+
if __name__=="__main__":
|
|
2591
|
+
annotate()
|
|
2592
|
+
|
|
2593
|
+
|
|
2594
|
+
|
|
2595
|
+
|
|
2596
|
+
|
|
2597
|
+
|
|
2598
|
+
|
|
2599
|
+
|
|
2600
|
+
|
|
2601
|
+
|
|
2602
|
+
|
|
2603
|
+
|
|
2604
|
+
|
|
2605
|
+
|
|
2606
|
+
|
|
2607
|
+
|
|
2608
|
+
|
|
2609
|
+
|
|
2610
|
+
|
|
2611
|
+
|
|
2612
|
+
|
|
2613
|
+
|
|
2614
|
+
|
|
2615
|
+
|
|
2616
|
+
|
|
2617
|
+
|
|
2618
|
+
|
|
2619
|
+
|
|
2620
|
+
|
|
2621
|
+
|
|
2622
|
+
|
|
2623
|
+
|
|
2624
|
+
|
|
2625
|
+
|
|
2626
|
+
|
|
2627
|
+
|
|
2628
|
+
|
|
2629
|
+
|
|
2630
|
+
|
|
2631
|
+
|
|
2632
|
+
|
|
2633
|
+
|
|
2634
|
+
|
|
2635
|
+
|
|
2636
|
+
|
|
2637
|
+
|
|
2638
|
+
|
|
2639
|
+
|
|
2640
|
+
|
|
2641
|
+
|
|
2642
|
+
|