networkvi 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- networkvi/__init__.py +32 -0
- networkvi/_constants.py +20 -0
- networkvi/_settings.py +220 -0
- networkvi/_types.py +13 -0
- networkvi/data/__init__.py +31 -0
- networkvi/data/_anntorchdataset.py +167 -0
- networkvi/data/_compat.py +174 -0
- networkvi/data/_constants.py +59 -0
- networkvi/data/_download.py +57 -0
- networkvi/data/_manager.py +539 -0
- networkvi/data/_preprocessing.py +473 -0
- networkvi/data/_read.py +72 -0
- networkvi/data/_utils.py +366 -0
- networkvi/data/fields/__init__.py +63 -0
- networkvi/data/fields/_arraylike_field.py +507 -0
- networkvi/data/fields/_base_field.py +152 -0
- networkvi/data/fields/_dataframe_field.py +284 -0
- networkvi/data/fields/_layer_field.py +148 -0
- networkvi/data/fields/_mudata.py +143 -0
- networkvi/data/fields/_protein.py +166 -0
- networkvi/data/fields/_scanvi.py +110 -0
- networkvi/data/fields/_uns_field.py +85 -0
- networkvi/dataloaders/__init__.py +15 -0
- networkvi/dataloaders/_ann_dataloader.py +136 -0
- networkvi/dataloaders/_data_splitting.py +210 -0
- networkvi/dataloaders/_samplers.py +72 -0
- networkvi/distributions/__init__.py +17 -0
- networkvi/distributions/_beta_binomial.py +130 -0
- networkvi/distributions/_constraints.py +41 -0
- networkvi/distributions/_negative_binomial.py +685 -0
- networkvi/distributions/_utils.py +46 -0
- networkvi/model/__init__.py +6 -0
- networkvi/model/_networkvi.py +2026 -0
- networkvi/model/_utils.py +270 -0
- networkvi/model/base/__init__.py +13 -0
- networkvi/model/base/_archesmixin_networkvi.py +293 -0
- networkvi/model/base/_base_model.py +940 -0
- networkvi/model/base/_constants.py +16 -0
- networkvi/model/base/_de_core.py +161 -0
- networkvi/model/base/_differential.py +724 -0
- networkvi/model/base/_log_likelihood.py +87 -0
- networkvi/model/base/_save_load.py +210 -0
- networkvi/model/base/_training_mixin.py +142 -0
- networkvi/model/base/_vaemixin.py +306 -0
- networkvi/module/__init__.py +7 -0
- networkvi/module/_classifier.py +78 -0
- networkvi/module/_constants.py +27 -0
- networkvi/module/_networkvae.py +1503 -0
- networkvi/module/_peakvae.py +362 -0
- networkvi/module/_utils.py +55 -0
- networkvi/module/base/__init__.py +14 -0
- networkvi/module/base/_base_module.py +332 -0
- networkvi/module/base/_decorators.py +132 -0
- networkvi/module/base/_embedding_mixin.py +55 -0
- networkvi/module/base/_pyro.py +11 -0
- networkvi/nn/__init__.py +21 -0
- networkvi/nn/_base_components.py +2289 -0
- networkvi/nn/_block_sparse_linear.py +490 -0
- networkvi/nn/_embedding.py +100 -0
- networkvi/nn/_evaluate_funcs.py +317 -0
- networkvi/nn/_gene_gene_interactions.py +98 -0
- networkvi/nn/_gene_models.py +448 -0
- networkvi/nn/_go_cov_attention.py +202 -0
- networkvi/nn/_go_embedding_layer.py +223 -0
- networkvi/nn/_go_graph.py +454 -0
- networkvi/nn/_go_utils.py +658 -0
- networkvi/nn/_introspections.py +873 -0
- networkvi/nn/_torch_utils.py +88 -0
- networkvi/nn/_utils.py +19 -0
- networkvi/train/__init__.py +19 -0
- networkvi/train/_callbacks.py +321 -0
- networkvi/train/_constants.py +17 -0
- networkvi/train/_logger.py +91 -0
- networkvi/train/_metrics.py +91 -0
- networkvi/train/_progress.py +98 -0
- networkvi/train/_trainer.py +195 -0
- networkvi/train/_trainingplans.py +692 -0
- networkvi/train/_trainrunner.py +146 -0
- networkvi/utils/__init__.py +17 -0
- networkvi/utils/_attrdict.py +12 -0
- networkvi/utils/_decorators.py +16 -0
- networkvi/utils/_dependencies.py +28 -0
- networkvi/utils/_docstrings.py +232 -0
- networkvi/utils/_jax.py +19 -0
- networkvi/utils/_track.py +58 -0
- networkvi-0.1.0.dist-info/METADATA +170 -0
- networkvi-0.1.0.dist-info/RECORD +88 -0
- networkvi-0.1.0.dist-info/WHEEL +4 -0
networkvi/__init__.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""scvi-tools."""
|
|
2
|
+
|
|
3
|
+
# Set default logging handler to avoid logging with logging.lastResort logger.
|
|
4
|
+
import logging
|
|
5
|
+
import warnings
|
|
6
|
+
|
|
7
|
+
from ._constants import REGISTRY_KEYS
|
|
8
|
+
from ._settings import settings
|
|
9
|
+
|
|
10
|
+
# this import needs to come after prior imports to prevent circular import
|
|
11
|
+
from . import data, model, utils
|
|
12
|
+
|
|
13
|
+
from importlib.metadata import version
|
|
14
|
+
|
|
15
|
+
package_name = "networkvi"
|
|
16
|
+
__version__ = "1.0.0"
|
|
17
|
+
|
|
18
|
+
settings.verbosity = logging.INFO
|
|
19
|
+
|
|
20
|
+
# Jax sets the root logger, this prevents double output.
|
|
21
|
+
scvi_logger = logging.getLogger("networkvi")
|
|
22
|
+
scvi_logger.propagate = False
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"settings",
|
|
27
|
+
"REGISTRY_KEYS",
|
|
28
|
+
"data",
|
|
29
|
+
"model",
|
|
30
|
+
"utils",
|
|
31
|
+
]
|
|
32
|
+
|
networkvi/_constants.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from typing import NamedTuple
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class _REGISTRY_KEYS_NT(NamedTuple):
|
|
5
|
+
X_KEY: str = "X"
|
|
6
|
+
BATCH_KEY: str = "batch"
|
|
7
|
+
PATIENT_KEY: str = "patient"
|
|
8
|
+
LABELS_KEY: str = "labels"
|
|
9
|
+
PROTEIN_EXP_KEY: str = "proteins"
|
|
10
|
+
CAT_COVS_KEY: str = "extra_categorical_covs"
|
|
11
|
+
CONT_COVS_KEY: str = "extra_continuous_covs"
|
|
12
|
+
INDICES_KEY: str = "ind_x"
|
|
13
|
+
SIZE_FACTOR_KEY: str = "size_factor"
|
|
14
|
+
MINIFY_TYPE_KEY: str = "minify_type"
|
|
15
|
+
LATENT_QZM_KEY: str = "latent_qzm"
|
|
16
|
+
LATENT_QZV_KEY: str = "latent_qzv"
|
|
17
|
+
OBSERVED_LIB_SIZE: str = "observed_lib_size"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
REGISTRY_KEYS = _REGISTRY_KEYS_NT()
|
networkvi/_settings.py
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Literal, Optional, Union
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
from lightning.pytorch import seed_everything
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.logging import RichHandler
|
|
10
|
+
|
|
11
|
+
scvi_logger = logging.getLogger("scvi")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ScviConfig:
|
|
15
|
+
"""Config manager for scvi-tools.
|
|
16
|
+
|
|
17
|
+
Examples
|
|
18
|
+
--------
|
|
19
|
+
To set the seed
|
|
20
|
+
|
|
21
|
+
>>> networkvi.settings.seed = 1
|
|
22
|
+
|
|
23
|
+
To set the batch size for functions like `networkvi.get_latent_representation`
|
|
24
|
+
|
|
25
|
+
>>> networkvi.settings.batch_size = 1024
|
|
26
|
+
|
|
27
|
+
To set the progress bar style, choose one of "rich", "tqdm"
|
|
28
|
+
|
|
29
|
+
>>> networkvi.settings.progress_bar_style = "rich"
|
|
30
|
+
|
|
31
|
+
To set the verbosity
|
|
32
|
+
|
|
33
|
+
>>> import logging
|
|
34
|
+
>>> networkvi.settings.verbosity = logging.INFO
|
|
35
|
+
|
|
36
|
+
To set the number of threads PyTorch will use
|
|
37
|
+
|
|
38
|
+
>>> networkvi.settings.num_threads = 2
|
|
39
|
+
|
|
40
|
+
To prevent Jax from preallocating GPU memory on start (default)
|
|
41
|
+
|
|
42
|
+
>>> networkvi.settings.jax_preallocate_gpu_memory = False
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
verbosity: int = logging.INFO,
|
|
48
|
+
progress_bar_style: Literal["rich", "tqdm"] = "tqdm",
|
|
49
|
+
batch_size: int = 128,
|
|
50
|
+
seed: Optional[int] = None,
|
|
51
|
+
logging_dir: str = "./scvi_log/",
|
|
52
|
+
dl_num_workers: int = 0,
|
|
53
|
+
jax_preallocate_gpu_memory: bool = False,
|
|
54
|
+
warnings_stacklevel: int = 2,
|
|
55
|
+
):
|
|
56
|
+
self.warnings_stacklevel = warnings_stacklevel
|
|
57
|
+
self.seed = seed
|
|
58
|
+
self.batch_size = batch_size
|
|
59
|
+
if progress_bar_style not in ["rich", "tqdm"]:
|
|
60
|
+
raise ValueError("Progress bar style must be in ['rich', 'tqdm']")
|
|
61
|
+
self.progress_bar_style = progress_bar_style
|
|
62
|
+
self.logging_dir = logging_dir
|
|
63
|
+
self.dl_num_workers = dl_num_workers
|
|
64
|
+
self._num_threads = None
|
|
65
|
+
self.jax_preallocate_gpu_memory = jax_preallocate_gpu_memory
|
|
66
|
+
self.verbosity = verbosity
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def batch_size(self) -> int:
|
|
70
|
+
"""Minibatch size for loading data into the model.
|
|
71
|
+
|
|
72
|
+
This is only used after a model is trained. Trainers have specific
|
|
73
|
+
`batch_size` parameters.
|
|
74
|
+
"""
|
|
75
|
+
return self._batch_size
|
|
76
|
+
|
|
77
|
+
@batch_size.setter
|
|
78
|
+
def batch_size(self, batch_size: int):
|
|
79
|
+
"""Minibatch size for loading data into the model.
|
|
80
|
+
|
|
81
|
+
This is only used after a model is trained. Trainers have specific
|
|
82
|
+
`batch_size` parameters.
|
|
83
|
+
"""
|
|
84
|
+
self._batch_size = batch_size
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def dl_num_workers(self) -> int:
|
|
88
|
+
"""Number of workers for PyTorch data loaders (Default is 0)."""
|
|
89
|
+
return self._dl_num_workers
|
|
90
|
+
|
|
91
|
+
@dl_num_workers.setter
|
|
92
|
+
def dl_num_workers(self, dl_num_workers: int):
|
|
93
|
+
"""Number of workers for PyTorch data loaders (Default is 0)."""
|
|
94
|
+
self._dl_num_workers = dl_num_workers
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def logging_dir(self) -> Path:
|
|
98
|
+
"""Directory for training logs (default `'./scvi_log/'`)."""
|
|
99
|
+
return self._logging_dir
|
|
100
|
+
|
|
101
|
+
@logging_dir.setter
|
|
102
|
+
def logging_dir(self, logging_dir: Union[str, Path]):
|
|
103
|
+
self._logging_dir = Path(logging_dir).resolve()
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def num_threads(self) -> None:
|
|
107
|
+
"""Number of threads PyTorch will use."""
|
|
108
|
+
return self._num_threads
|
|
109
|
+
|
|
110
|
+
@num_threads.setter
|
|
111
|
+
def num_threads(self, num: int):
|
|
112
|
+
"""Number of threads PyTorch will use."""
|
|
113
|
+
self._num_threads = num
|
|
114
|
+
torch.set_num_threads(num)
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def progress_bar_style(self) -> str:
|
|
118
|
+
"""Library to use for progress bar."""
|
|
119
|
+
return self._pbar_style
|
|
120
|
+
|
|
121
|
+
@progress_bar_style.setter
|
|
122
|
+
def progress_bar_style(self, pbar_style: Literal["tqdm", "rich"]):
|
|
123
|
+
"""Library to use for progress bar."""
|
|
124
|
+
self._pbar_style = pbar_style
|
|
125
|
+
|
|
126
|
+
@property
|
|
127
|
+
def seed(self) -> int:
|
|
128
|
+
"""Random seed for torch and numpy."""
|
|
129
|
+
return self._seed
|
|
130
|
+
|
|
131
|
+
@seed.setter
|
|
132
|
+
def seed(self, seed: Union[int, None] = None):
|
|
133
|
+
"""Random seed for torch and numpy."""
|
|
134
|
+
if seed is None:
|
|
135
|
+
self._seed = None
|
|
136
|
+
else:
|
|
137
|
+
torch.backends.cudnn.deterministic = True
|
|
138
|
+
torch.backends.cudnn.benchmark = False
|
|
139
|
+
seed_everything(seed)
|
|
140
|
+
self._seed = seed
|
|
141
|
+
|
|
142
|
+
@property
|
|
143
|
+
def verbosity(self) -> int:
|
|
144
|
+
"""Verbosity level (default `logging.INFO`)."""
|
|
145
|
+
return self._verbosity
|
|
146
|
+
|
|
147
|
+
@verbosity.setter
|
|
148
|
+
def verbosity(self, level: Union[str, int]):
|
|
149
|
+
"""Sets logging configuration for scvi based on chosen level of verbosity.
|
|
150
|
+
|
|
151
|
+
If "scvi" logger has no StreamHandler, add one.
|
|
152
|
+
Else, set its level to `level`.
|
|
153
|
+
|
|
154
|
+
Parameters
|
|
155
|
+
----------
|
|
156
|
+
level
|
|
157
|
+
Sets "scvi" logging level to `level`
|
|
158
|
+
force_terminal
|
|
159
|
+
Rich logging option, set to False if piping to file output.
|
|
160
|
+
"""
|
|
161
|
+
self._verbosity = level
|
|
162
|
+
scvi_logger.setLevel(level)
|
|
163
|
+
if len(scvi_logger.handlers) == 0:
|
|
164
|
+
console = Console(force_terminal=True)
|
|
165
|
+
if console.is_jupyter is True:
|
|
166
|
+
console.is_jupyter = False
|
|
167
|
+
ch = RichHandler(level=level, show_path=False, console=console, show_time=False)
|
|
168
|
+
formatter = logging.Formatter("%(message)s")
|
|
169
|
+
ch.setFormatter(formatter)
|
|
170
|
+
scvi_logger.addHandler(ch)
|
|
171
|
+
else:
|
|
172
|
+
scvi_logger.setLevel(level)
|
|
173
|
+
|
|
174
|
+
@property
|
|
175
|
+
def warnings_stacklevel(self) -> int:
|
|
176
|
+
"""Stacklevel for warnings."""
|
|
177
|
+
return self._warnings_stacklevel
|
|
178
|
+
|
|
179
|
+
@warnings_stacklevel.setter
|
|
180
|
+
def warnings_stacklevel(self, stacklevel: int):
|
|
181
|
+
"""Stacklevel for warnings."""
|
|
182
|
+
self._warnings_stacklevel = stacklevel
|
|
183
|
+
|
|
184
|
+
def reset_logging_handler(self):
|
|
185
|
+
"""Resets "scvi" log handler to a basic RichHandler().
|
|
186
|
+
|
|
187
|
+
This is useful if piping outputs to a file.
|
|
188
|
+
"""
|
|
189
|
+
scvi_logger.removeHandler(scvi_logger.handlers[0])
|
|
190
|
+
ch = RichHandler(level=self._verbosity, show_path=False, show_time=False)
|
|
191
|
+
formatter = logging.Formatter("%(message)s")
|
|
192
|
+
ch.setFormatter(formatter)
|
|
193
|
+
scvi_logger.addHandler(ch)
|
|
194
|
+
|
|
195
|
+
@property
|
|
196
|
+
def jax_preallocate_gpu_memory(self):
|
|
197
|
+
"""Jax GPU memory allocation settings.
|
|
198
|
+
|
|
199
|
+
If False, Jax will ony preallocate GPU memory it needs.
|
|
200
|
+
If float in (0, 1), Jax will preallocate GPU memory to that
|
|
201
|
+
fraction of the GPU memory.
|
|
202
|
+
"""
|
|
203
|
+
return self._jax_gpu
|
|
204
|
+
|
|
205
|
+
@jax_preallocate_gpu_memory.setter
|
|
206
|
+
def jax_preallocate_gpu_memory(self, value: Union[float, bool]):
|
|
207
|
+
# see https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html#gpu-memory-allocation
|
|
208
|
+
if value is False:
|
|
209
|
+
os.environ["XLA_PYTHON_CLIENT_PREALLOCATE"] = "false"
|
|
210
|
+
elif isinstance(value, float):
|
|
211
|
+
if value >= 1 or value <= 0:
|
|
212
|
+
raise ValueError("Need to use a value between 0 and 1")
|
|
213
|
+
# format is ".XX"
|
|
214
|
+
os.environ["XLA_PYTHON_CLIENT_MEM_FRACTION"] = str(value)[1:4]
|
|
215
|
+
else:
|
|
216
|
+
raise ValueError("value not understood, need bool or float in (0, 1)")
|
|
217
|
+
self._jax_gpu = value
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
settings = ScviConfig()
|
networkvi/_types.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from typing import Literal, Union
|
|
2
|
+
|
|
3
|
+
import anndata
|
|
4
|
+
import jax.numpy as jnp
|
|
5
|
+
import mudata
|
|
6
|
+
import torch
|
|
7
|
+
|
|
8
|
+
Number = Union[int, float]
|
|
9
|
+
AnnOrMuData = Union[anndata.AnnData, mudata.MuData]
|
|
10
|
+
Tensor = Union[torch.Tensor, jnp.ndarray]
|
|
11
|
+
LossRecord = Union[dict[str, Tensor], Tensor]
|
|
12
|
+
# TODO(adamgayoso): Add constants for minified data types.
|
|
13
|
+
MinifiedDataType = Literal["latent_posterior_parameters"]
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from anndata import read_csv, read_h5ad, read_loom, read_text
|
|
2
|
+
|
|
3
|
+
from ._anntorchdataset import AnnTorchDataset
|
|
4
|
+
from ._manager import AnnDataManager, AnnDataManagerValidationCheck
|
|
5
|
+
from ._preprocessing import (
|
|
6
|
+
add_dna_sequence,
|
|
7
|
+
organize_cite_seq_10x,
|
|
8
|
+
organize_multiome_anndatas,
|
|
9
|
+
poisson_gene_selection,
|
|
10
|
+
reads_to_fragments,
|
|
11
|
+
)
|
|
12
|
+
from ._read import read_10x_atac, read_10x_multiome
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"AnnTorchDataset",
|
|
16
|
+
"AnnDataManagerValidationCheck",
|
|
17
|
+
"AnnDataManager",
|
|
18
|
+
"poisson_gene_selection",
|
|
19
|
+
"organize_cite_seq_10x",
|
|
20
|
+
"read_h5ad",
|
|
21
|
+
"read_csv",
|
|
22
|
+
"read_loom",
|
|
23
|
+
"read_text",
|
|
24
|
+
"read_10x_atac",
|
|
25
|
+
"read_10x_multiome",
|
|
26
|
+
"organize_multiome_anndatas",
|
|
27
|
+
"add_dna_sequence",
|
|
28
|
+
"reads_to_fragments",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
import h5py
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pandas as pd
|
|
9
|
+
import torch
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
# anndata >= 0.10
|
|
13
|
+
from anndata.experimental import CSCDataset, CSRDataset
|
|
14
|
+
|
|
15
|
+
SparseDataset = (CSRDataset, CSCDataset)
|
|
16
|
+
except ImportError:
|
|
17
|
+
from anndata._core.sparse_dataset import SparseDataset
|
|
18
|
+
|
|
19
|
+
from scipy.sparse import issparse
|
|
20
|
+
from torch.utils.data import Dataset
|
|
21
|
+
|
|
22
|
+
from networkvi._constants import REGISTRY_KEYS
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
from ._manager import AnnDataManager
|
|
26
|
+
from ._utils import registry_key_to_default_dtype, scipy_to_torch_sparse
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
class AnnTorchDataset(Dataset):
|
|
31
|
+
"""Extension of :class:`~torch.utils.data.Dataset` for :class:`~anndata.AnnData` objects.
|
|
32
|
+
|
|
33
|
+
Parameters
|
|
34
|
+
----------
|
|
35
|
+
adata_manager
|
|
36
|
+
:class:`~networkvi.data.AnnDataManager` object with a registered AnnData object.
|
|
37
|
+
getitem_tensors
|
|
38
|
+
Specifies the keys in the data registry (``adata_manager.data_registry``) to return in
|
|
39
|
+
``__getitem__``. One of the following:
|
|
40
|
+
|
|
41
|
+
* ``dict``: Keys correspond to keys in the data registry and values correspond to the
|
|
42
|
+
desired :class:`~np.dtype` of the returned data.
|
|
43
|
+
* ``list``: Elements correspond to keys in the data registry. Continuous data will be
|
|
44
|
+
returned as :class:`~np.float32` and discrete data will be returned as :class:`~np.int64`.
|
|
45
|
+
* ``None``: All registered data will be returned. Continuous data will be returned as
|
|
46
|
+
:class:`~np.float32` and discrete data will be returned as :class:`~np.int64`.
|
|
47
|
+
load_sparse_tensor
|
|
48
|
+
``EXPERIMENTAL`` If ``True``, loads data with sparse CSR or CSC layout as a
|
|
49
|
+
:class:`~torch.Tensor` with the same layout. Can lead to speedups in data transfers to
|
|
50
|
+
GPUs, depending on the sparsity of the data.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
adata_manager: AnnDataManager,
|
|
56
|
+
getitem_tensors: list | dict[str, type] | None = None,
|
|
57
|
+
load_sparse_tensor: bool = False,
|
|
58
|
+
):
|
|
59
|
+
super().__init__()
|
|
60
|
+
|
|
61
|
+
if adata_manager.adata is None:
|
|
62
|
+
raise ValueError("Please run ``register_fields`` on ``adata_manager`` first.")
|
|
63
|
+
self.adata_manager = adata_manager
|
|
64
|
+
self.keys_and_dtypes = getitem_tensors
|
|
65
|
+
self.load_sparse_tensor = load_sparse_tensor
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def registered_keys(self):
|
|
69
|
+
"""Keys in the data registry."""
|
|
70
|
+
return self.adata_manager.data_registry.keys()
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def keys_and_dtypes(self):
|
|
74
|
+
"""Keys and corresponding :class:`~np.dtype` of data to fetch in ``__getitem__``."""
|
|
75
|
+
return self._keys_and_dtypes
|
|
76
|
+
|
|
77
|
+
@keys_and_dtypes.setter
|
|
78
|
+
def keys_and_dtypes(self, getitem_tensors: list | dict[str, type] | None):
|
|
79
|
+
"""Set keys and corresponding :class:`~np.dtype` of data to fetch in ``__getitem__``.
|
|
80
|
+
|
|
81
|
+
Raises an error if any of the keys are not in the data registry.
|
|
82
|
+
"""
|
|
83
|
+
if isinstance(getitem_tensors, list):
|
|
84
|
+
keys_to_dtypes = {key: registry_key_to_default_dtype(key) for key in getitem_tensors}
|
|
85
|
+
elif isinstance(getitem_tensors, dict):
|
|
86
|
+
keys_to_dtypes = getitem_tensors
|
|
87
|
+
elif getitem_tensors is None:
|
|
88
|
+
keys_to_dtypes = {
|
|
89
|
+
key: registry_key_to_default_dtype(key) for key in self.registered_keys
|
|
90
|
+
}
|
|
91
|
+
else:
|
|
92
|
+
raise ValueError("`getitem_tensors` must be a `list`, `dict`, or `None`")
|
|
93
|
+
|
|
94
|
+
for key in keys_to_dtypes:
|
|
95
|
+
if key not in self.registered_keys:
|
|
96
|
+
raise KeyError(f"{key} not found in the data registry.")
|
|
97
|
+
|
|
98
|
+
self._keys_and_dtypes = keys_to_dtypes
|
|
99
|
+
|
|
100
|
+
@property
|
|
101
|
+
def data(self):
|
|
102
|
+
"""Dictionary of data tensors.
|
|
103
|
+
|
|
104
|
+
First time this is accessed, data is fetched from the underlying
|
|
105
|
+
:class:`~anndata.AnnData` object. Subsequent accesses will return the
|
|
106
|
+
cached dictionary.
|
|
107
|
+
"""
|
|
108
|
+
if not hasattr(self, "_data"):
|
|
109
|
+
self._data = {
|
|
110
|
+
key: self.adata_manager.get_from_registry(key) for key in self.keys_and_dtypes
|
|
111
|
+
}
|
|
112
|
+
return self._data
|
|
113
|
+
|
|
114
|
+
def __len__(self):
|
|
115
|
+
return self.adata_manager.adata.shape[0]
|
|
116
|
+
|
|
117
|
+
def __getitem__(
|
|
118
|
+
self, indexes: int | list[int] | slice
|
|
119
|
+
) -> dict[str, np.ndarray | torch.Tensor]:
|
|
120
|
+
"""Fetch data from the :class:`~anndata.AnnData` object.
|
|
121
|
+
|
|
122
|
+
Parameters
|
|
123
|
+
----------
|
|
124
|
+
indexes
|
|
125
|
+
Indexes of the observations to fetch. Can be a single index, a list of indexes, or a
|
|
126
|
+
slice.
|
|
127
|
+
|
|
128
|
+
Returns
|
|
129
|
+
-------
|
|
130
|
+
Mapping of data registry keys to arrays of shape ``(n_obs, ...)``.
|
|
131
|
+
"""
|
|
132
|
+
if isinstance(indexes, int):
|
|
133
|
+
indexes = [indexes] # force batched single observations
|
|
134
|
+
|
|
135
|
+
if self.adata_manager.adata.isbacked and isinstance(indexes, (list, np.ndarray)):
|
|
136
|
+
# need to sort indexes for h5py datasets
|
|
137
|
+
indexes = np.sort(indexes)
|
|
138
|
+
|
|
139
|
+
data_map = {}
|
|
140
|
+
|
|
141
|
+
for key, dtype in self.keys_and_dtypes.items():
|
|
142
|
+
data = self.data[key]
|
|
143
|
+
|
|
144
|
+
if isinstance(data, (np.ndarray, h5py.Dataset)):
|
|
145
|
+
sliced_data = data[indexes].astype(dtype, copy=False)
|
|
146
|
+
elif isinstance(data, pd.DataFrame):
|
|
147
|
+
sliced_data = data.iloc[indexes, :].to_numpy().astype(dtype, copy=False)
|
|
148
|
+
elif issparse(data) or isinstance(data, SparseDataset):
|
|
149
|
+
sliced_data = data[indexes].astype(dtype, copy=False)
|
|
150
|
+
if self.load_sparse_tensor:
|
|
151
|
+
sliced_data = scipy_to_torch_sparse(sliced_data)
|
|
152
|
+
else:
|
|
153
|
+
sliced_data = sliced_data.toarray()
|
|
154
|
+
elif issparse(data) or isinstance(data, da.Array):
|
|
155
|
+
sliced_data = data[indexes].compute()
|
|
156
|
+
elif isinstance(data, str) and key == REGISTRY_KEYS.MINIFY_TYPE_KEY:
|
|
157
|
+
# for minified anndata, we need this because we can have a string
|
|
158
|
+
# for `data``, which is the value of the MINIFY_TYPE_KEY in adata.uns,
|
|
159
|
+
# used to record the type data minification
|
|
160
|
+
# TODO: Adata manager should have a list of which fields it will load
|
|
161
|
+
continue
|
|
162
|
+
else:
|
|
163
|
+
raise TypeError(f"{key} is not a supported type")
|
|
164
|
+
|
|
165
|
+
data_map[key] = sliced_data
|
|
166
|
+
|
|
167
|
+
return data_map
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
from copy import deepcopy
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
|
|
6
|
+
from networkvi import REGISTRY_KEYS
|
|
7
|
+
|
|
8
|
+
from . import _constants
|
|
9
|
+
from .fields import (
|
|
10
|
+
CategoricalJointObsField,
|
|
11
|
+
CategoricalObsField,
|
|
12
|
+
LabelsWithUnlabeledObsField,
|
|
13
|
+
LayerField,
|
|
14
|
+
NumericalJointObsField,
|
|
15
|
+
ProteinObsmField,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
LEGACY_REGISTRY_KEY_MAP = {
|
|
19
|
+
"X": REGISTRY_KEYS.X_KEY,
|
|
20
|
+
"batch_indices": REGISTRY_KEYS.BATCH_KEY,
|
|
21
|
+
"labels": REGISTRY_KEYS.LABELS_KEY,
|
|
22
|
+
"cat_covs": REGISTRY_KEYS.CAT_COVS_KEY,
|
|
23
|
+
"cont_covs": REGISTRY_KEYS.CONT_COVS_KEY,
|
|
24
|
+
"protein_expression": REGISTRY_KEYS.PROTEIN_EXP_KEY,
|
|
25
|
+
"ind_x": REGISTRY_KEYS.INDICES_KEY,
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _infer_setup_args(model_cls, setup_dict: dict, unlabeled_category: Optional[str]) -> dict:
|
|
30
|
+
setup_args = {}
|
|
31
|
+
data_registry = setup_dict[_constants._DATA_REGISTRY_KEY]
|
|
32
|
+
categorical_mappings = setup_dict["categorical_mappings"]
|
|
33
|
+
for registry_key, adata_mapping in data_registry.items():
|
|
34
|
+
if registry_key not in LEGACY_REGISTRY_KEY_MAP:
|
|
35
|
+
continue
|
|
36
|
+
new_registry_key = LEGACY_REGISTRY_KEY_MAP[registry_key]
|
|
37
|
+
|
|
38
|
+
attr_name = adata_mapping[_constants._DR_ATTR_NAME]
|
|
39
|
+
attr_key = adata_mapping[_constants._DR_ATTR_KEY]
|
|
40
|
+
if attr_name == _constants._ADATA_ATTRS.X:
|
|
41
|
+
setup_args["layer"] = None
|
|
42
|
+
elif attr_name == _constants._ADATA_ATTRS.LAYERS:
|
|
43
|
+
setup_args["layer"] = attr_key
|
|
44
|
+
elif attr_name == _constants._ADATA_ATTRS.OBS:
|
|
45
|
+
if new_registry_key in {REGISTRY_KEYS.BATCH_KEY, REGISTRY_KEYS.LABELS_KEY}:
|
|
46
|
+
# Omit labels field for TOTALVI
|
|
47
|
+
if (
|
|
48
|
+
model_cls.__name__ == "TOTALVI"
|
|
49
|
+
and new_registry_key == REGISTRY_KEYS.LABELS_KEY
|
|
50
|
+
):
|
|
51
|
+
continue
|
|
52
|
+
setup_args[f"{new_registry_key}_key"] = categorical_mappings[attr_key][
|
|
53
|
+
"original_key"
|
|
54
|
+
]
|
|
55
|
+
elif attr_name == _constants._ADATA_ATTRS.OBSM:
|
|
56
|
+
if new_registry_key == REGISTRY_KEYS.CONT_COVS_KEY:
|
|
57
|
+
obs_keys = setup_dict["extra_continuous_keys"]
|
|
58
|
+
setup_args["continuous_covariate_keys"] = obs_keys
|
|
59
|
+
elif new_registry_key == REGISTRY_KEYS.CAT_COVS_KEY:
|
|
60
|
+
obs_keys = setup_dict["extra_categoricals"]["keys"]
|
|
61
|
+
setup_args["categorical_covariate_keys"] = obs_keys
|
|
62
|
+
elif new_registry_key == REGISTRY_KEYS.PROTEIN_EXP_KEY:
|
|
63
|
+
setup_args["protein_expression_obsm_key"] = attr_key
|
|
64
|
+
setup_args["protein_names_uns_key"] = "_protein_names"
|
|
65
|
+
|
|
66
|
+
if unlabeled_category is not None:
|
|
67
|
+
setup_args["unlabeled_category"] = unlabeled_category
|
|
68
|
+
|
|
69
|
+
return {
|
|
70
|
+
_constants._MODEL_NAME_KEY: model_cls.__name__,
|
|
71
|
+
_constants._SETUP_ARGS_KEY: setup_args,
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def registry_from_setup_dict(
|
|
76
|
+
model_cls, setup_dict: dict, unlabeled_category: Optional[str] = None
|
|
77
|
+
) -> dict:
|
|
78
|
+
"""Converts old setup dict format to new registry dict format.
|
|
79
|
+
|
|
80
|
+
Only to be used for backwards compatibility when loading setup dictionaries for models.
|
|
81
|
+
Takes old hard-coded setup dictionary structure and fills in the analogous registry structure.
|
|
82
|
+
|
|
83
|
+
Parameters
|
|
84
|
+
----------
|
|
85
|
+
model_cls
|
|
86
|
+
Model class used for setup.
|
|
87
|
+
setup_dict
|
|
88
|
+
Setup dictionary created after registering an AnnData with former ``setup_anndata``
|
|
89
|
+
implementation.
|
|
90
|
+
unlabeled_category
|
|
91
|
+
Unlabeled category value used in :class:`~networkvi.model.SCANVI`.
|
|
92
|
+
"""
|
|
93
|
+
registry = {
|
|
94
|
+
_constants._SCVI_VERSION_KEY: setup_dict[_constants._SCVI_VERSION_KEY],
|
|
95
|
+
_constants._FIELD_REGISTRIES_KEY: {},
|
|
96
|
+
}
|
|
97
|
+
data_registry = setup_dict[_constants._DATA_REGISTRY_KEY]
|
|
98
|
+
categorical_mappings = setup_dict["categorical_mappings"]
|
|
99
|
+
summary_stats = setup_dict[_constants._SUMMARY_STATS_KEY]
|
|
100
|
+
field_registries = registry[_constants._FIELD_REGISTRIES_KEY]
|
|
101
|
+
for (
|
|
102
|
+
registry_key,
|
|
103
|
+
adata_mapping,
|
|
104
|
+
) in data_registry.items(): # Note: this does not work for empty fields.
|
|
105
|
+
if registry_key not in LEGACY_REGISTRY_KEY_MAP:
|
|
106
|
+
continue
|
|
107
|
+
new_registry_key = LEGACY_REGISTRY_KEY_MAP[registry_key]
|
|
108
|
+
|
|
109
|
+
attr_name = adata_mapping[_constants._DR_ATTR_NAME]
|
|
110
|
+
attr_key = adata_mapping[_constants._DR_ATTR_KEY]
|
|
111
|
+
if attr_key is not None and attr_key.startswith("_scvi"):
|
|
112
|
+
adata_mapping[_constants._DR_ATTR_KEY] = f"_scvi_{new_registry_key}"
|
|
113
|
+
|
|
114
|
+
field_registries[new_registry_key] = {
|
|
115
|
+
_constants._DATA_REGISTRY_KEY: adata_mapping,
|
|
116
|
+
_constants._STATE_REGISTRY_KEY: {},
|
|
117
|
+
_constants._SUMMARY_STATS_KEY: {},
|
|
118
|
+
}
|
|
119
|
+
field_registry = field_registries[new_registry_key]
|
|
120
|
+
field_state_registry = field_registry[_constants._STATE_REGISTRY_KEY]
|
|
121
|
+
field_summary_stats = field_registry[_constants._SUMMARY_STATS_KEY]
|
|
122
|
+
|
|
123
|
+
if attr_name in (_constants._ADATA_ATTRS.X, _constants._ADATA_ATTRS.LAYERS):
|
|
124
|
+
field_state_registry[LayerField.N_VARS_KEY] = summary_stats["n_vars"]
|
|
125
|
+
field_summary_stats.update(field_state_registry)
|
|
126
|
+
elif attr_name == _constants._ADATA_ATTRS.OBS:
|
|
127
|
+
categorical_mapping = categorical_mappings[attr_key]
|
|
128
|
+
# Default labels field for TOTALVI
|
|
129
|
+
if model_cls.__name__ == "TOTALVI" and new_registry_key == REGISTRY_KEYS.LABELS_KEY:
|
|
130
|
+
field_state_registry[CategoricalObsField.CATEGORICAL_MAPPING_KEY] = np.zeros(
|
|
131
|
+
1, dtype=np.int64
|
|
132
|
+
)
|
|
133
|
+
else:
|
|
134
|
+
field_state_registry[CategoricalObsField.CATEGORICAL_MAPPING_KEY] = (
|
|
135
|
+
categorical_mapping["mapping"]
|
|
136
|
+
)
|
|
137
|
+
field_state_registry[CategoricalObsField.ORIGINAL_ATTR_KEY] = categorical_mapping[
|
|
138
|
+
"original_key"
|
|
139
|
+
]
|
|
140
|
+
if new_registry_key == REGISTRY_KEYS.BATCH_KEY:
|
|
141
|
+
field_summary_stats[f"n_{new_registry_key}"] = summary_stats["n_batch"]
|
|
142
|
+
elif new_registry_key == REGISTRY_KEYS.LABELS_KEY:
|
|
143
|
+
field_summary_stats[f"n_{new_registry_key}"] = summary_stats["n_labels"]
|
|
144
|
+
if unlabeled_category is not None:
|
|
145
|
+
field_state_registry[LabelsWithUnlabeledObsField.UNLABELED_CATEGORY] = (
|
|
146
|
+
unlabeled_category
|
|
147
|
+
)
|
|
148
|
+
elif attr_name == _constants._ADATA_ATTRS.OBSM:
|
|
149
|
+
if new_registry_key == REGISTRY_KEYS.CONT_COVS_KEY:
|
|
150
|
+
columns = setup_dict["extra_continuous_keys"].copy()
|
|
151
|
+
field_state_registry[NumericalJointObsField.COLUMNS_KEY] = columns
|
|
152
|
+
field_summary_stats[f"n_{new_registry_key}"] = columns.shape[0]
|
|
153
|
+
elif new_registry_key == REGISTRY_KEYS.CAT_COVS_KEY:
|
|
154
|
+
extra_categoricals_mapping = deepcopy(setup_dict["extra_categoricals"])
|
|
155
|
+
field_state_registry.update(extra_categoricals_mapping)
|
|
156
|
+
field_state_registry[CategoricalJointObsField.FIELD_KEYS_KEY] = (
|
|
157
|
+
field_state_registry.pop("keys")
|
|
158
|
+
)
|
|
159
|
+
field_summary_stats[f"n_{new_registry_key}"] = len(
|
|
160
|
+
extra_categoricals_mapping["keys"]
|
|
161
|
+
)
|
|
162
|
+
elif new_registry_key == REGISTRY_KEYS.PROTEIN_EXP_KEY:
|
|
163
|
+
field_state_registry[ProteinObsmField.COLUMN_NAMES_KEY] = setup_dict[
|
|
164
|
+
"protein_names"
|
|
165
|
+
].copy()
|
|
166
|
+
if "totalvi_batch_mask" in setup_dict:
|
|
167
|
+
field_state_registry[ProteinObsmField.PROTEIN_BATCH_MASK] = setup_dict[
|
|
168
|
+
"totalvi_batch_mask"
|
|
169
|
+
].copy()
|
|
170
|
+
field_summary_stats[f"n_{new_registry_key}"] = len(setup_dict["protein_names"])
|
|
171
|
+
|
|
172
|
+
registry.update(_infer_setup_args(model_cls, setup_dict, unlabeled_category))
|
|
173
|
+
|
|
174
|
+
return registry
|