scnmf 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scnmf/__init__.py +29 -0
- scnmf/_version.py +24 -0
- scnmf/cli.py +163 -0
- scnmf/config.py +102 -0
- scnmf/data/anndata_io.py +167 -0
- scnmf/data/batch_sampler.py +171 -0
- scnmf/data/sparse_csr.py +141 -0
- scnmf/evaluation/batch_metrics.py +82 -0
- scnmf/evaluation/reconstruction.py +26 -0
- scnmf/models/cluster_head.py +94 -0
- scnmf/models/conditional_nmf.py +364 -0
- scnmf/models/factors.py +358 -0
- scnmf/models/likelihoods.py +70 -0
- scnmf/training/logging.py +97 -0
- scnmf/training/losses.py +83 -0
- scnmf/training/trainer.py +309 -0
- scnmf-0.1.0.dist-info/METADATA +142 -0
- scnmf-0.1.0.dist-info/RECORD +20 -0
- scnmf-0.1.0.dist-info/WHEEL +4 -0
- scnmf-0.1.0.dist-info/entry_points.txt +2 -0
scnmf/__init__.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Sparse conditional joint NMF for single-cell RNA-seq."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
|
|
7
|
+
from scnmf.config import SCNMFConfig
|
|
8
|
+
from scnmf.data.batch_sampler import partition_from_labels
|
|
9
|
+
from scnmf.data.sparse_csr import SparseBatch, SparseExpressionMatrix
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"__version__",
|
|
13
|
+
"SCNMFConfig",
|
|
14
|
+
"SparseConditionalNMF",
|
|
15
|
+
"SparseExpressionMatrix",
|
|
16
|
+
"SparseBatch",
|
|
17
|
+
"partition_from_labels",
|
|
18
|
+
"setup_anndata",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def __getattr__(name: str):
|
|
23
|
+
if name == "SparseConditionalNMF":
|
|
24
|
+
from scnmf.models.conditional_nmf import SparseConditionalNMF
|
|
25
|
+
return SparseConditionalNMF
|
|
26
|
+
if name == "setup_anndata":
|
|
27
|
+
from scnmf.data.anndata_io import setup_anndata
|
|
28
|
+
return setup_anndata
|
|
29
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
scnmf/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.1.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 1, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
scnmf/cli.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Command-line interface for the ``scnmf`` package."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
from typing import Any, Dict, Optional
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
from scnmf.config import SCNMFConfig
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _read_json(path: str) -> Dict[str, Any]:
|
|
17
|
+
with open(path, "r", encoding="utf-8") as handle:
|
|
18
|
+
return json.load(handle)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
22
|
+
parser = argparse.ArgumentParser(
|
|
23
|
+
prog="scnmf",
|
|
24
|
+
description="Sparse conditional joint NMF for single-cell RNA-seq integration and analysis.",
|
|
25
|
+
)
|
|
26
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
27
|
+
|
|
28
|
+
base = argparse.ArgumentParser(add_help=False)
|
|
29
|
+
base.add_argument("--adata", required=True, help="Input AnnData .h5ad file")
|
|
30
|
+
base.add_argument("--batch-key", default="batch", help="Column in adata.obs defining batches")
|
|
31
|
+
base.add_argument("--layer", default=None, help="Count layer to use; defaults to adata.X")
|
|
32
|
+
base.add_argument("--output", required=True, help="Output directory")
|
|
33
|
+
base.add_argument("--device", default="auto", help="Device: auto, cpu, or cuda")
|
|
34
|
+
base.add_argument("--seed", type=int, default=0)
|
|
35
|
+
|
|
36
|
+
train = subparsers.add_parser("train", parents=[base], help="Train a conditional joint NMF model")
|
|
37
|
+
train.add_argument("--config", help="JSON configuration file")
|
|
38
|
+
train.add_argument("--n-shared", type=int, default=30)
|
|
39
|
+
train.add_argument("--n-conditional", type=int, default=8)
|
|
40
|
+
train.add_argument("--batch-size", type=int, default=2048)
|
|
41
|
+
train.add_argument("--max-epochs", type=int, default=200)
|
|
42
|
+
train.add_argument("--learning-rate", type=float, default=0.005)
|
|
43
|
+
train.add_argument("--likelihood", default="poisson", choices=["poisson", "kl", "gaussian"])
|
|
44
|
+
train.add_argument("--verbose/--silent", default=True, action=argparse.BooleanOptionalAction)
|
|
45
|
+
|
|
46
|
+
transform = subparsers.add_parser("transform", parents=[base], help="Project cells with a trained model")
|
|
47
|
+
transform.add_argument("--checkpoint", required=True, help="Directory containing model.pt and metadata.json")
|
|
48
|
+
transform.add_argument("--chunk-size", type=int, default=4096)
|
|
49
|
+
|
|
50
|
+
cluster = subparsers.add_parser("cluster", parents=[base], help="Cluster shared loadings")
|
|
51
|
+
cluster.add_argument("--method", default="kmeans", choices=["kmeans", "leiden"])
|
|
52
|
+
cluster.add_argument("--n-clusters", type=int, default=20)
|
|
53
|
+
cluster.add_argument("--resolution", type=float, default=0.8)
|
|
54
|
+
cluster.add_argument("--key", default="scnmf_cluster")
|
|
55
|
+
|
|
56
|
+
evaluate = subparsers.add_parser("evaluate", parents=[base], help="Write evaluation metadata")
|
|
57
|
+
evaluate.add_argument("--metrics", default="integration", choices=["integration", "reconstruction", "all"])
|
|
58
|
+
evaluate.add_argument("--true-labels", default=None, help="Optional obs column with ground-truth labels")
|
|
59
|
+
|
|
60
|
+
return parser
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _load_anndata(path: str):
|
|
64
|
+
try:
|
|
65
|
+
from anndata import read_h5ad
|
|
66
|
+
except ImportError as exc: # pragma: no cover - environment-dependent
|
|
67
|
+
raise SystemExit("AnnData is required: pip install 'scnmf[anndata]'") from exc
|
|
68
|
+
return read_h5ad(path)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _build_config(args: argparse.Namespace) -> SCNMFConfig:
|
|
72
|
+
values: Dict[str, Any] = {}
|
|
73
|
+
if getattr(args, "config", None):
|
|
74
|
+
values.update(_read_json(args.config))
|
|
75
|
+
for key in {
|
|
76
|
+
"n_shared",
|
|
77
|
+
"n_conditional",
|
|
78
|
+
"likelihood",
|
|
79
|
+
"batch_size",
|
|
80
|
+
"max_epochs",
|
|
81
|
+
"learning_rate",
|
|
82
|
+
"seed",
|
|
83
|
+
"verbose",
|
|
84
|
+
}:
|
|
85
|
+
if hasattr(args, key):
|
|
86
|
+
values.setdefault(key, getattr(args, key))
|
|
87
|
+
return SCNMFConfig.from_dict(values)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def command_train(args: argparse.Namespace) -> int:
|
|
91
|
+
from scnmf.models.conditional_nmf import SparseConditionalNMF
|
|
92
|
+
|
|
93
|
+
adata = _load_anndata(args.adata)
|
|
94
|
+
config = _build_config(args)
|
|
95
|
+
model = SparseConditionalNMF(config=config, device=args.device)
|
|
96
|
+
model.fit(
|
|
97
|
+
adata,
|
|
98
|
+
layer=args.layer,
|
|
99
|
+
batch_key=args.batch_key,
|
|
100
|
+
use_raw=False,
|
|
101
|
+
)
|
|
102
|
+
model.save(args.output)
|
|
103
|
+
model.annotate(adata).write(os.path.join(args.output, "annotated.h5ad"), compression="gzip")
|
|
104
|
+
return 0
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def command_transform(args: argparse.Namespace) -> int:
|
|
108
|
+
from scnmf.models.conditional_nmf import SparseConditionalNMF
|
|
109
|
+
|
|
110
|
+
model = SparseConditionalNMF(device=args.device).load(args.checkpoint)
|
|
111
|
+
adata = _load_anndata(args.adata)
|
|
112
|
+
result = model.transform(adata, layer=args.layer, chunk_size=args.chunk_size)
|
|
113
|
+
np.savez_compressed(os.path.join(args.output, "loadings.npz"), **{k: v for k, v in result.items() if isinstance(v, np.ndarray)})
|
|
114
|
+
return 0
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def command_cluster(args: argparse.Namespace) -> int:
|
|
118
|
+
adata = _load_anndata(args.adata)
|
|
119
|
+
key = "scnmf_shared"
|
|
120
|
+
if key not in adata.obsm:
|
|
121
|
+
raise SystemExit(f"{key} not found in adata.obsm; run train or transform first.")
|
|
122
|
+
loadings = np.asarray(adata.obsm[key], dtype=np.float32)
|
|
123
|
+
from scnmf.models.cluster_head import KMeansClusterHead, scanpy_leiden_labels
|
|
124
|
+
|
|
125
|
+
if args.method == "kmeans":
|
|
126
|
+
labels = KMeansClusterHead(n_clusters=args.n_clusters, random_state=args.seed).fit(loadings)
|
|
127
|
+
else:
|
|
128
|
+
labels = scanpy_leiden_labels(loadings, resolution=args.resolution, random_state=args.seed)
|
|
129
|
+
adata.obs[args.key] = np.asarray(labels, dtype=np.intp).astype("category")
|
|
130
|
+
adata.write(os.path.join(args.output, "clustered.h5ad"), compression="gzip")
|
|
131
|
+
return 0
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def command_evaluate(args: argparse.Namespace) -> int:
|
|
135
|
+
from scnmf.evaluation.batch_metrics import evaluate_loadings
|
|
136
|
+
|
|
137
|
+
adata = _load_anndata(args.adata)
|
|
138
|
+
if "scnmf_shared" not in adata.obsm:
|
|
139
|
+
raise SystemExit("scnmf_shared embedding is required for evaluation")
|
|
140
|
+
loadings = np.asarray(adata.obsm["scnmf_shared"], dtype=np.float32)
|
|
141
|
+
batch_labels = np.asarray(adata.obs[args.batch_key].astype(str)) if args.batch_key in adata.obs else None
|
|
142
|
+
true_labels = np.asarray(adata.obs[args.true_labels].astype(str)) if args.true_labels and args.true_labels in adata.obs else None
|
|
143
|
+
result = evaluate_loadings(loadings, batch_labels=batch_labels, true_labels=true_labels, predicted_labels=None)
|
|
144
|
+
os.makedirs(args.output, exist_ok=True)
|
|
145
|
+
with open(os.path.join(args.output, "evaluation.json"), "w", encoding="utf-8") as handle:
|
|
146
|
+
json.dump(result, handle, indent=2)
|
|
147
|
+
return 0
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
151
|
+
parser = build_parser()
|
|
152
|
+
args = parser.parse_args(argv)
|
|
153
|
+
commands = {
|
|
154
|
+
"train": command_train,
|
|
155
|
+
"transform": command_transform,
|
|
156
|
+
"cluster": command_cluster,
|
|
157
|
+
"evaluate": command_evaluate,
|
|
158
|
+
}
|
|
159
|
+
return commands[args.command](args)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
if __name__ == "__main__":
|
|
163
|
+
sys.exit(main())
|
scnmf/config.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Typed, serializable configuration for :class:`scnmf.models.conditional_nmf.SparseConditionalNMF`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import dataclasses
|
|
6
|
+
import json
|
|
7
|
+
from dataclasses import dataclass, field, fields
|
|
8
|
+
from typing import Any, Dict, Optional
|
|
9
|
+
|
|
10
|
+
LIKELIHOODS = {"gaussian", "kl", "poisson"}
|
|
11
|
+
CONSTRAINTS = {"softplus", "exp", "clamp"}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _positive_integer(name: str, value: int) -> None:
|
|
15
|
+
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
|
16
|
+
raise ValueError(f"{name} must be a positive integer, got {value!r}")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _nonnegative_float(name: str, value: float) -> None:
|
|
20
|
+
if not isinstance(value, (int, float)) or isinstance(value, bool) or value < 0:
|
|
21
|
+
raise ValueError(f"{name} must be a non-negative real number, got {value!r}")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class SCNMFConfig:
|
|
26
|
+
"""Training and model configuration.
|
|
27
|
+
|
|
28
|
+
Attributes are deliberately simple JSON types so that configurations can be
|
|
29
|
+
written to model manifests and reused from command-line tools.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
n_shared: int = 30
|
|
33
|
+
n_conditional: int = 8
|
|
34
|
+
likelihood: str = "poisson"
|
|
35
|
+
constraint: str = "softplus"
|
|
36
|
+
library: str = "fixed"
|
|
37
|
+
use_batch_norm: bool = True
|
|
38
|
+
shared_l1: float = 0.0
|
|
39
|
+
conditional_l1: float = 0.0
|
|
40
|
+
loading_l2: float = 0.01
|
|
41
|
+
batch_size: int = 2048
|
|
42
|
+
max_epochs: int = 200
|
|
43
|
+
learning_rate: float = 0.005
|
|
44
|
+
optimizer: str = "adamw"
|
|
45
|
+
weight_decay: float = 0.0
|
|
46
|
+
warmup_epochs: int = 0
|
|
47
|
+
grad_clip: Optional[float] = 1.0
|
|
48
|
+
batch_balance: str = "stratified"
|
|
49
|
+
min_cells_per_batch: int = 16
|
|
50
|
+
max_cells_per_batch: Optional[int] = None
|
|
51
|
+
use_sparse_tensor: bool = False
|
|
52
|
+
eps: float = 1e-8
|
|
53
|
+
seed: int = 0
|
|
54
|
+
verbose: bool = True
|
|
55
|
+
log_interval: int = 25
|
|
56
|
+
extra: Dict[str, Any] = field(default_factory=dict)
|
|
57
|
+
|
|
58
|
+
def __post_init__(self) -> None:
|
|
59
|
+
_positive_integer("n_shared", self.n_shared)
|
|
60
|
+
if self.n_conditional < 0:
|
|
61
|
+
raise ValueError("n_conditional must be non-negative")
|
|
62
|
+
if self.likelihood not in LIKELIHOODS:
|
|
63
|
+
raise ValueError(f"likelihood must be one of {sorted(LIKELIHOODS)}")
|
|
64
|
+
if self.constraint not in CONSTRAINTS:
|
|
65
|
+
raise ValueError(f"constraint must be one of {sorted(CONSTRAINTS)}")
|
|
66
|
+
if self.library not in {"fixed", "shared", "cell"}:
|
|
67
|
+
raise ValueError("library must be 'fixed', 'shared', or 'cell'")
|
|
68
|
+
_positive_integer("batch_size", self.batch_size)
|
|
69
|
+
_positive_integer("max_epochs", self.max_epochs)
|
|
70
|
+
_nonnegative_float("learning_rate", self.learning_rate)
|
|
71
|
+
if self.optimizer not in {"adam", "adamw", "sgd"}:
|
|
72
|
+
raise ValueError("optimizer must be 'adam', 'adamw', or 'sgd'")
|
|
73
|
+
_nonnegative_float("weight_decay", self.weight_decay)
|
|
74
|
+
_nonnegative_float("loading_l2", self.loading_l2)
|
|
75
|
+
_positive_integer("log_interval", self.log_interval)
|
|
76
|
+
_nonnegative_float("eps", self.eps)
|
|
77
|
+
|
|
78
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
79
|
+
return dataclasses.asdict(self)
|
|
80
|
+
|
|
81
|
+
def to_json(self) -> str:
|
|
82
|
+
return json.dumps(self.to_dict(), sort_keys=True, indent=2)
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def from_dict(cls, data: Dict[str, Any]) -> "SCNMFConfig":
|
|
86
|
+
unknown = set(data) - {field.name for field in fields(cls)}
|
|
87
|
+
if unknown:
|
|
88
|
+
raise ValueError(f"Unknown configuration keys: {sorted(unknown)}")
|
|
89
|
+
return cls(**data)
|
|
90
|
+
|
|
91
|
+
def get(self, key: str, default: Any = None) -> Any:
|
|
92
|
+
if key in self.extra:
|
|
93
|
+
return self.extra[key]
|
|
94
|
+
if hasattr(self, key):
|
|
95
|
+
return getattr(self, key)
|
|
96
|
+
return default
|
|
97
|
+
|
|
98
|
+
def replace(self, **changes: Any) -> "SCNMFConfig":
|
|
99
|
+
values = self.to_dict()
|
|
100
|
+
values.update(changes)
|
|
101
|
+
values.pop("extra", None)
|
|
102
|
+
return SCNMFConfig.from_dict(values)
|
scnmf/data/anndata_io.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""AnnData ingestion, validation, and result persistence."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Dict, Optional, Sequence
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
from scnmf.data.sparse_csr import SparseExpressionMatrix
|
|
11
|
+
|
|
12
|
+
try:
|
|
13
|
+
import anndata
|
|
14
|
+
_HAS_ANNDATA = True
|
|
15
|
+
except ImportError: # pragma: no cover - optional dependency
|
|
16
|
+
anndata = None
|
|
17
|
+
_HAS_ANNDATA = False
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class AnnDataNotInstalledError(ImportError):
|
|
21
|
+
"""Raised when AnnData functionality is requested but the package is absent."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _require_anndata():
|
|
25
|
+
if not _HAS_ANNDATA:
|
|
26
|
+
raise AnnDataNotInstalledError(
|
|
27
|
+
"AnnData support requires the 'anndata' package. "
|
|
28
|
+
"Install scnmf with 'pip install \"scnmf[anndata]\"'."
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class AnnDataView:
|
|
34
|
+
"""Subset-safe description of where to read counts and batch labels."""
|
|
35
|
+
|
|
36
|
+
layer: Optional[str]
|
|
37
|
+
batch_key: str
|
|
38
|
+
gene_mask_key: Optional[str]
|
|
39
|
+
use_raw: bool
|
|
40
|
+
n_cells: int
|
|
41
|
+
n_genes: int
|
|
42
|
+
batch_levels: tuple
|
|
43
|
+
batch_sizes: Dict[str, int]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def validate_anndata(
|
|
47
|
+
adata,
|
|
48
|
+
layer: Optional[str] = None,
|
|
49
|
+
batch_key: str = "batch",
|
|
50
|
+
gene_mask_key: Optional[str] = None,
|
|
51
|
+
use_raw: bool = False,
|
|
52
|
+
require_batch: bool = True,
|
|
53
|
+
) -> AnnDataView:
|
|
54
|
+
"""Return a validated view of an AnnData object."""
|
|
55
|
+
_require_anndata()
|
|
56
|
+
if not isinstance(adata, anndata.AnnData):
|
|
57
|
+
raise TypeError(f"Expected an AnnData object, got {type(adata).__name__}")
|
|
58
|
+
|
|
59
|
+
counts = adata.raw.X if use_raw and adata.raw is not None else adata.X
|
|
60
|
+
if layer is not None:
|
|
61
|
+
if layer not in adata.layers:
|
|
62
|
+
raise KeyError(f"Layer {layer!r} is not present in adata.layers")
|
|
63
|
+
counts = adata.layers[layer]
|
|
64
|
+
|
|
65
|
+
if np.issubdtype(np.asarray(counts).dtype, np.bool_):
|
|
66
|
+
raise TypeError("Boolean expression matrices are not supported")
|
|
67
|
+
|
|
68
|
+
if batch_key not in adata.obs:
|
|
69
|
+
if require_batch:
|
|
70
|
+
raise KeyError(f"Batch key {batch_key!r} is not present in adata.obs")
|
|
71
|
+
batch_levels = ("__single_batch__",)
|
|
72
|
+
batch_labels = np.array([batch_levels[0]] * adata.n_obs)
|
|
73
|
+
else:
|
|
74
|
+
batch_labels = np.asarray(adata.obs[batch_key].astype(str))
|
|
75
|
+
batch_levels = tuple(sorted(set(batch_labels)))
|
|
76
|
+
|
|
77
|
+
if gene_mask_key is not None:
|
|
78
|
+
if gene_mask_key not in adata.var.columns:
|
|
79
|
+
raise KeyError(f"Gene mask {gene_mask_key!r} is not present in adata.var")
|
|
80
|
+
mask = np.asarray(adata.var[gene_mask_key]).ravel().astype(bool)
|
|
81
|
+
if mask.shape != (adata.n_vars,):
|
|
82
|
+
raise ValueError("Gene mask must have one value per gene")
|
|
83
|
+
if not mask.any():
|
|
84
|
+
raise ValueError("Gene mask must contain at least one True value")
|
|
85
|
+
adata = adata[:, mask]
|
|
86
|
+
|
|
87
|
+
return AnnDataView(
|
|
88
|
+
layer=layer,
|
|
89
|
+
batch_key=batch_key,
|
|
90
|
+
gene_mask_key=gene_mask_key,
|
|
91
|
+
use_raw=use_raw,
|
|
92
|
+
n_cells=int(adata.n_obs),
|
|
93
|
+
n_genes=int(adata.n_vars),
|
|
94
|
+
batch_levels=batch_levels,
|
|
95
|
+
batch_sizes={level: int((batch_labels == level).sum()) for level in batch_levels},
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def create_expression_matrix(adata, view: AnnDataView) -> SparseExpressionMatrix:
|
|
100
|
+
"""Materialize a non-negative CSR expression matrix from an AnnData object."""
|
|
101
|
+
_require_anndata()
|
|
102
|
+
if view.layer is not None and view.layer not in adata.layers:
|
|
103
|
+
raise KeyError(f"Layer {view.layer!r} is not present in adata.layers")
|
|
104
|
+
if view.use_raw and adata.raw is not None:
|
|
105
|
+
counts = adata.raw.X
|
|
106
|
+
genes = np.asarray(adata.raw.var_names)
|
|
107
|
+
else:
|
|
108
|
+
counts = adata.X if view.layer is None else adata.layers[view.layer]
|
|
109
|
+
genes = np.asarray(adata.var_names)
|
|
110
|
+
if view.gene_mask_key is not None:
|
|
111
|
+
mask = np.asarray(adata.var[view.gene_mask_key]).ravel().astype(bool)
|
|
112
|
+
counts = counts[:, mask]
|
|
113
|
+
genes = genes[mask]
|
|
114
|
+
|
|
115
|
+
from scipy import sparse
|
|
116
|
+
|
|
117
|
+
matrix = sparse.csr_matrix(counts, dtype=np.float32)
|
|
118
|
+
matrix.eliminate_zeros()
|
|
119
|
+
if matrix.min() < 0:
|
|
120
|
+
raise ValueError("Expression matrix contains negative values")
|
|
121
|
+
|
|
122
|
+
return SparseExpressionMatrix(
|
|
123
|
+
matrix=matrix,
|
|
124
|
+
row_names=np.asarray(adata.obs_names),
|
|
125
|
+
col_names=np.asarray(genes, dtype=object),
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def write_results(
|
|
130
|
+
adata,
|
|
131
|
+
*,
|
|
132
|
+
shared_loadings: np.ndarray,
|
|
133
|
+
conditional_loadings: Optional[np.ndarray],
|
|
134
|
+
shared_programs: np.ndarray,
|
|
135
|
+
conditional_programs: Optional[np.ndarray],
|
|
136
|
+
batch_labels: np.ndarray,
|
|
137
|
+
metadata: Optional[Dict] = None,
|
|
138
|
+
shared_key: str = "scnmf_shared",
|
|
139
|
+
conditional_key: str = "scnmf_conditional",
|
|
140
|
+
program_key: str = "scnmf_shared_programs",
|
|
141
|
+
) -> None:
|
|
142
|
+
"""Attach loadings, programs, and provenance metadata to an AnnData object."""
|
|
143
|
+
_require_anndata()
|
|
144
|
+
if shared_loadings.shape[0] != adata.n_obs:
|
|
145
|
+
raise ValueError("Shared loadings must have one row per cell")
|
|
146
|
+
if shared_programs.shape[0] != adata.n_vars:
|
|
147
|
+
raise ValueError("Shared programs must have one row per gene")
|
|
148
|
+
if conditional_loadings is not None and conditional_loadings.shape[0] != adata.n_obs:
|
|
149
|
+
raise ValueError("Conditional loadings must have one row per cell")
|
|
150
|
+
|
|
151
|
+
adata.obsm[shared_key] = np.asarray(shared_loadings, dtype=np.float32)
|
|
152
|
+
adata.varm[program_key] = np.asarray(shared_programs, dtype=np.float32)
|
|
153
|
+
if conditional_loadings is not None:
|
|
154
|
+
adata.obsm[conditional_key] = np.asarray(conditional_loadings, dtype=np.float32)
|
|
155
|
+
if conditional_programs is not None:
|
|
156
|
+
adata.varm["scnmf_conditional_programs"] = np.asarray(
|
|
157
|
+
conditional_programs, dtype=np.float32
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
record = {
|
|
161
|
+
"version": "0.1.0",
|
|
162
|
+
"batch_key": metadata.get("batch_key") if metadata else None,
|
|
163
|
+
"batch_labels": [str(label) for label in batch_labels],
|
|
164
|
+
"config": metadata.get("config") if metadata else None,
|
|
165
|
+
"history": metadata.get("history", []) if metadata else [],
|
|
166
|
+
}
|
|
167
|
+
adata.uns.setdefault("scnmf", {}).update(record)
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""Stratified batch sampling for imbalanced single-cell batches."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Dict, Iterator, List, Optional, Sequence, Tuple
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class BatchPartition:
|
|
14
|
+
"""Cells grouped by a categorical batch or sampling stratum."""
|
|
15
|
+
|
|
16
|
+
labels: np.ndarray
|
|
17
|
+
groups: Dict[str, np.ndarray]
|
|
18
|
+
sizes: Dict[str, int]
|
|
19
|
+
order: Tuple[str, ...]
|
|
20
|
+
|
|
21
|
+
@property
|
|
22
|
+
def n_groups(self) -> int:
|
|
23
|
+
return len(self.order)
|
|
24
|
+
|
|
25
|
+
@property
|
|
26
|
+
def n_cells(self) -> int:
|
|
27
|
+
return len(self.labels)
|
|
28
|
+
|
|
29
|
+
def __post_init__(self) -> None:
|
|
30
|
+
for key, members in self.groups.items():
|
|
31
|
+
if members.ndim != 1:
|
|
32
|
+
raise ValueError(f"Group {key} must be a 1-D index array")
|
|
33
|
+
if members.size and (members.min() < 0 or members.max() >= self.n_cells):
|
|
34
|
+
raise ValueError(f"Group {key} contains out-of-range cell indices")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def partition_from_labels(labels: Sequence[str]) -> BatchPartition:
|
|
38
|
+
"""Construct a :class:`BatchPartition` from an ordered label sequence."""
|
|
39
|
+
labels = np.asarray(list(labels))
|
|
40
|
+
groups: Dict[str, List[int]] = {}
|
|
41
|
+
for index, label in enumerate(labels):
|
|
42
|
+
groups.setdefault(str(label), []).append(index)
|
|
43
|
+
ordered = tuple(sorted(groups))
|
|
44
|
+
arrays = {key: np.asarray(groups[key], dtype=np.intp) for key in ordered}
|
|
45
|
+
sizes = {key: int(arrays[key].size) for key in ordered}
|
|
46
|
+
return BatchPartition(labels=labels, groups=arrays, sizes=sizes, order=ordered)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def partition_by_labels(labels: Sequence[str]) -> BatchPartition:
|
|
50
|
+
"""Build index partitions from an array-like label sequence."""
|
|
51
|
+
labels = np.asarray(labels)
|
|
52
|
+
if labels.ndim != 1:
|
|
53
|
+
raise ValueError("labels must be a one-dimensional sequence")
|
|
54
|
+
if labels.size == 0:
|
|
55
|
+
raise ValueError("labels cannot be empty")
|
|
56
|
+
groups: Dict[str, List[int]] = {}
|
|
57
|
+
for index, label in enumerate(labels):
|
|
58
|
+
groups.setdefault(str(label), []).append(index)
|
|
59
|
+
ordered = tuple(sorted(groups))
|
|
60
|
+
arrays = {key: np.asarray(groups[key], dtype=np.intp) for key in ordered}
|
|
61
|
+
sizes = {key: int(arrays[key].size) for key in ordered}
|
|
62
|
+
return BatchPartition(labels=labels, groups=arrays, sizes=sizes, order=ordered)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class StratifiedBatchSampler:
|
|
66
|
+
"""Yield mini-batches that contain a configurable number of cells per batch label.
|
|
67
|
+
|
|
68
|
+
The sampler first selects the number of cells requested for each batch label,
|
|
69
|
+
then samples *within* each label using a common generator. When the requested
|
|
70
|
+
number exceeds the label size, sampling is performed with replacement for that
|
|
71
|
+
label only; otherwise sampling is without replacement.
|
|
72
|
+
|
|
73
|
+
Parameters
|
|
74
|
+
----------
|
|
75
|
+
partition
|
|
76
|
+
Precomputed cell partitions.
|
|
77
|
+
batch_size
|
|
78
|
+
Desired total number of cells per mini-batch. Must be positive.
|
|
79
|
+
min_cells_per_batch, max_cells_per_batch
|
|
80
|
+
Floor and ceiling applied to each label's allocation before sampling.
|
|
81
|
+
replacement
|
|
82
|
+
When ``False``, labels whose allocation exceeds their population trigger
|
|
83
|
+
an error instead of sampling with replacement.
|
|
84
|
+
seed
|
|
85
|
+
Optional seed for reproducible sampling.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
def __init__(
|
|
89
|
+
self,
|
|
90
|
+
partition: BatchPartition,
|
|
91
|
+
batch_size: int,
|
|
92
|
+
min_cells_per_batch: int = 16,
|
|
93
|
+
max_cells_per_batch: Optional[int] = None,
|
|
94
|
+
replacement: bool = True,
|
|
95
|
+
seed: Optional[int] = None,
|
|
96
|
+
) -> None:
|
|
97
|
+
if batch_size <= 0:
|
|
98
|
+
raise ValueError("batch_size must be positive")
|
|
99
|
+
if min_cells_per_batch < 1:
|
|
100
|
+
raise ValueError("min_cells_per_batch must be at least 1")
|
|
101
|
+
if partition.n_groups == 0:
|
|
102
|
+
raise ValueError("partition must contain at least one group")
|
|
103
|
+
if batch_size < partition.n_groups * min_cells_per_batch:
|
|
104
|
+
raise ValueError(
|
|
105
|
+
"batch_size is too small to satisfy min_cells_per_batch for every group"
|
|
106
|
+
)
|
|
107
|
+
self.partition = partition
|
|
108
|
+
self.batch_size = batch_size
|
|
109
|
+
self.min_cells = min_cells_per_batch
|
|
110
|
+
self.max_cells = max_cells_per_batch
|
|
111
|
+
self.replacement = replacement
|
|
112
|
+
self._generator = np.random.default_rng(seed)
|
|
113
|
+
|
|
114
|
+
def _allocate(self) -> Dict[str, int]:
|
|
115
|
+
groups = self.partition.order
|
|
116
|
+
remaining = self.batch_size
|
|
117
|
+
allocations: Dict[str, int] = {}
|
|
118
|
+
for group in groups:
|
|
119
|
+
desired = max(self.min_cells, remaining // (len(groups) - len(allocations)))
|
|
120
|
+
if self.max_cells is not None:
|
|
121
|
+
desired = min(desired, self.max_cells)
|
|
122
|
+
size = self.partition.sizes[group]
|
|
123
|
+
allocations[group] = min(desired, size)
|
|
124
|
+
remaining -= allocations[group]
|
|
125
|
+
# Redistribute any leftover quota to groups that have not reached their cap.
|
|
126
|
+
while remaining > 0:
|
|
127
|
+
moved = False
|
|
128
|
+
for group in groups:
|
|
129
|
+
cap = self.max_cells if self.max_cells is not None else self.partition.sizes[group]
|
|
130
|
+
if allocations[group] < cap:
|
|
131
|
+
allocations[group] += 1
|
|
132
|
+
remaining -= 1
|
|
133
|
+
moved = True
|
|
134
|
+
if not moved:
|
|
135
|
+
break
|
|
136
|
+
return allocations
|
|
137
|
+
|
|
138
|
+
def sample_batch(self) -> np.ndarray:
|
|
139
|
+
allocations = self._allocate()
|
|
140
|
+
chosen: List[np.ndarray] = []
|
|
141
|
+
for group, count in allocations.items():
|
|
142
|
+
members = self.partition.groups[group]
|
|
143
|
+
if count >= members.size:
|
|
144
|
+
if not self.replacement and count > members.size:
|
|
145
|
+
raise ValueError(
|
|
146
|
+
f"Requested {count} cells from batch {group} but only "
|
|
147
|
+
f"{members.size} are available and replacement=False"
|
|
148
|
+
)
|
|
149
|
+
chosen.append(self._generator.choice(members, size=count, replace=self.replacement))
|
|
150
|
+
else:
|
|
151
|
+
chosen.append(self._generator.choice(members, size=count, replace=False))
|
|
152
|
+
batch = np.concatenate(chosen)
|
|
153
|
+
self._generator.shuffle(batch)
|
|
154
|
+
return batch
|
|
155
|
+
|
|
156
|
+
def __iter__(self) -> Iterator[np.ndarray]:
|
|
157
|
+
while True:
|
|
158
|
+
yield self.sample_batch()
|
|
159
|
+
|
|
160
|
+
def batches_per_epoch(self, n_cells: Optional[int] = None) -> int:
|
|
161
|
+
"""Round requested batch size to the nearest whole number."""
|
|
162
|
+
n_cells = self.partition.n_cells if n_cells is None else n_cells
|
|
163
|
+
if n_cells <= 0:
|
|
164
|
+
raise ValueError("n_cells must be positive")
|
|
165
|
+
return max(1, math.ceil(n_cells / self.batch_size))
|
|
166
|
+
|
|
167
|
+
def state_dict(self) -> dict:
|
|
168
|
+
return {"bit_generator": self._generator.bit_generator.state}
|
|
169
|
+
|
|
170
|
+
def load_state_dict(self, state: dict) -> None:
|
|
171
|
+
self._generator.bit_generator.state = state["bit_generator"]
|