pyadata-cli 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- adata/__init__.py +10 -0
- adata/cli.py +1139 -0
- adata/commands/__init__.py +8 -0
- adata/commands/concat.py +21 -0
- adata/commands/create.py +93 -0
- adata/commands/export.py +105 -0
- adata/commands/import_data.py +144 -0
- adata/commands/info.py +199 -0
- adata/commands/ls.py +114 -0
- adata/commands/split.py +152 -0
- adata/commands/subset.py +17 -0
- adata/core/__init__.py +1 -0
- adata/core/concat.py +1084 -0
- adata/core/info.py +276 -0
- adata/core/query.py +268 -0
- adata/core/read.py +28 -0
- adata/core/select.py +122 -0
- adata/core/subset.py +767 -0
- adata/elements/__init__.py +10 -0
- adata/elements/read.py +310 -0
- adata/elements/spec.py +125 -0
- adata/elements/strings.py +90 -0
- adata/elements/write.py +255 -0
- adata/formats/__init__.py +1 -0
- adata/formats/array.py +109 -0
- adata/formats/common.py +51 -0
- adata/formats/dataframe.py +234 -0
- adata/formats/image.py +85 -0
- adata/formats/json_data.py +196 -0
- adata/formats/sparse.py +253 -0
- adata/formats/validate.py +180 -0
- adata/info.py +3 -0
- adata/py.typed +0 -0
- adata/read.py +19 -0
- adata/storage/__init__.py +529 -0
- adata/util/__init__.py +1 -0
- adata/util/path.py +9 -0
- pyadata_cli-0.5.0.dist-info/METADATA +135 -0
- pyadata_cli-0.5.0.dist-info/RECORD +42 -0
- pyadata_cli-0.5.0.dist-info/WHEEL +4 -0
- pyadata_cli-0.5.0.dist-info/entry_points.txt +4 -0
- pyadata_cli-0.5.0.dist-info/licenses/LICENSE +21 -0
adata/core/subset.py
ADDED
|
@@ -0,0 +1,767 @@
|
|
|
1
|
+
"""Subset operations for .h5ad and .zarr stores."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import shutil
|
|
7
|
+
from typing import Optional, Set, Tuple, List, Dict, Any
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
from rich.console import Console
|
|
11
|
+
from rich.progress import (
|
|
12
|
+
Progress,
|
|
13
|
+
SpinnerColumn,
|
|
14
|
+
TextColumn,
|
|
15
|
+
BarColumn,
|
|
16
|
+
TaskProgressColumn,
|
|
17
|
+
TimeElapsedColumn,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
from adata.elements import spec
|
|
21
|
+
from adata.elements.write import set_shape_attr, write_mapping
|
|
22
|
+
from adata.core.select import read_names, select_indices
|
|
23
|
+
from adata.elements.read import (
|
|
24
|
+
decode_str_array,
|
|
25
|
+
element_len,
|
|
26
|
+
read_str_chunk,
|
|
27
|
+
resolve_index,
|
|
28
|
+
)
|
|
29
|
+
from adata.storage import (
|
|
30
|
+
create_dataset,
|
|
31
|
+
copy_attrs,
|
|
32
|
+
copy_tree,
|
|
33
|
+
dataset_create_kwargs,
|
|
34
|
+
detect_backend,
|
|
35
|
+
is_dataset,
|
|
36
|
+
is_group,
|
|
37
|
+
is_zarr_group,
|
|
38
|
+
is_zarr_array,
|
|
39
|
+
open_store,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _target_backend(dst_group: Any) -> str:
|
|
44
|
+
return "zarr" if is_zarr_group(dst_group) else "hdf5"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _ensure_group(parent: Any, name: str) -> Any:
|
|
48
|
+
"""Get or create an AnnData mapping group, tagged `encoding-type: dict`.
|
|
49
|
+
|
|
50
|
+
Every container here (layers, obsm, obsp, varm, varp) is a mapping in the
|
|
51
|
+
spec; leaving one untagged makes anndata fall back to its legacy reader
|
|
52
|
+
and emit an OldFormatWarning.
|
|
53
|
+
"""
|
|
54
|
+
return write_mapping(parent, name)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _group_get(parent: Any, key: str) -> Any | None:
|
|
58
|
+
return parent[key] if key in parent else None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _ensure_optional_anndata_groups(dst: Any) -> None:
|
|
62
|
+
"""Create the optional mapping groups an AnnData store is expected to have."""
|
|
63
|
+
for key in ("layers", "obsm", "obsp", "varm", "varp"):
|
|
64
|
+
_ensure_group(dst, key)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _decode_attr(value: Any) -> Any:
|
|
68
|
+
if isinstance(value, bytes):
|
|
69
|
+
return value.decode("utf-8")
|
|
70
|
+
return value
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _read_name_file(path: Path) -> Set[str]:
|
|
74
|
+
names: Set[str] = set()
|
|
75
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
76
|
+
for line in f:
|
|
77
|
+
line = line.strip()
|
|
78
|
+
if line:
|
|
79
|
+
names.add(line)
|
|
80
|
+
return names
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def indices_from_name_set(
|
|
84
|
+
names_ds: Any,
|
|
85
|
+
keep: Set[str],
|
|
86
|
+
*,
|
|
87
|
+
chunk_size: int = 200_000,
|
|
88
|
+
) -> Tuple[np.ndarray, Set[str]]:
|
|
89
|
+
"""Resolve a set of names to sorted row indices, streaming the index.
|
|
90
|
+
|
|
91
|
+
`names_ds` may be a dataset or -- as anndata >= 0.11 writes indices -- a
|
|
92
|
+
`nullable-string-array` group, so it is read through
|
|
93
|
+
:func:`adata.elements.read.read_str_chunk` rather than sliced directly.
|
|
94
|
+
Returns the indices found and the names that were not.
|
|
95
|
+
"""
|
|
96
|
+
flat_len = element_len(names_ds)
|
|
97
|
+
|
|
98
|
+
remaining = set(keep)
|
|
99
|
+
found_indices: List[int] = []
|
|
100
|
+
cache: Dict[str, np.ndarray] = {}
|
|
101
|
+
|
|
102
|
+
for start in range(0, flat_len, chunk_size):
|
|
103
|
+
end = min(start + chunk_size, flat_len)
|
|
104
|
+
for i, name in enumerate(read_str_chunk(names_ds, start, end, cache)):
|
|
105
|
+
if name in remaining:
|
|
106
|
+
found_indices.append(start + i)
|
|
107
|
+
remaining.remove(name)
|
|
108
|
+
|
|
109
|
+
if not remaining:
|
|
110
|
+
break
|
|
111
|
+
|
|
112
|
+
return np.asarray(found_indices, dtype=np.int64), remaining
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _take_rows(obj: Any, indices: Optional[np.ndarray]) -> Any:
|
|
116
|
+
"""Read the selected rows of a dataset, handling both backends' indexing."""
|
|
117
|
+
if indices is None:
|
|
118
|
+
return obj[...]
|
|
119
|
+
if is_zarr_array(obj):
|
|
120
|
+
if obj.ndim == 1:
|
|
121
|
+
return obj.oindex[indices]
|
|
122
|
+
return obj.oindex[(indices,) + (slice(None),) * (obj.ndim - 1)]
|
|
123
|
+
return obj[indices, ...]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _copy_rows(
|
|
127
|
+
src_ds: Any,
|
|
128
|
+
dst_parent: Any,
|
|
129
|
+
name: str,
|
|
130
|
+
indices: Optional[np.ndarray],
|
|
131
|
+
) -> Any:
|
|
132
|
+
"""Write the selected rows of `src_ds` into `dst_parent` as `name`."""
|
|
133
|
+
if indices is None:
|
|
134
|
+
return copy_tree(src_ds, dst_parent, name)
|
|
135
|
+
|
|
136
|
+
target_backend = _target_backend(dst_parent)
|
|
137
|
+
kw = dataset_create_kwargs(
|
|
138
|
+
src_ds, target_backend=target_backend, dst_parent=dst_parent
|
|
139
|
+
)
|
|
140
|
+
kw = _clamp_chunks(kw, len(indices))
|
|
141
|
+
ds = create_dataset(
|
|
142
|
+
dst_parent,
|
|
143
|
+
name,
|
|
144
|
+
data=_take_rows(src_ds, indices),
|
|
145
|
+
**kw,
|
|
146
|
+
)
|
|
147
|
+
copy_attrs(src_ds.attrs, ds.attrs, target_backend=target_backend)
|
|
148
|
+
return ds
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _clamp_chunks(kw: dict, *out_shape: int) -> dict:
|
|
152
|
+
"""Shrink a forwarded chunk shape to fit the subset's dimensions.
|
|
153
|
+
|
|
154
|
+
h5py rejects a chunk larger than the dataset, so a chunked source subset
|
|
155
|
+
below its own chunk size would otherwise fail outright.
|
|
156
|
+
|
|
157
|
+
Zarr additionally requires a shard to be a whole number of chunks, so a
|
|
158
|
+
clamped chunk invalidates the source's shard geometry. Sharding is a
|
|
159
|
+
storage-layout choice rather than data, so it is dropped and left to the
|
|
160
|
+
backend rather than recomputed into something arbitrary.
|
|
161
|
+
"""
|
|
162
|
+
chunks = kw.get("chunks")
|
|
163
|
+
if not isinstance(chunks, (tuple, list)) or not chunks:
|
|
164
|
+
return kw
|
|
165
|
+
|
|
166
|
+
original = tuple(int(c) for c in chunks)
|
|
167
|
+
clamped = tuple(
|
|
168
|
+
min(c, out_shape[i]) if i < len(out_shape) and out_shape[i] > 0 else c
|
|
169
|
+
for i, c in enumerate(original)
|
|
170
|
+
)
|
|
171
|
+
if clamped == original:
|
|
172
|
+
return kw
|
|
173
|
+
|
|
174
|
+
kw = dict(kw)
|
|
175
|
+
kw["chunks"] = clamped
|
|
176
|
+
kw.pop("shards", None)
|
|
177
|
+
return kw
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def subset_axis_group(
|
|
181
|
+
src: Any,
|
|
182
|
+
dst: Any,
|
|
183
|
+
indices: Optional[np.ndarray],
|
|
184
|
+
) -> None:
|
|
185
|
+
"""Copy a dataframe group, taking only `indices` along its rows.
|
|
186
|
+
|
|
187
|
+
Every column layout the spec allows has to be narrowed here, not just plain
|
|
188
|
+
datasets: `categorical` keeps its categories and subsets only `codes`,
|
|
189
|
+
while the masked layouts (`nullable-*`, which anndata >= 0.11 uses for the
|
|
190
|
+
index and every string column) subset both `values` and `mask`. Copying a
|
|
191
|
+
masked column whole would leave it longer than the rest of the frame.
|
|
192
|
+
"""
|
|
193
|
+
target_backend = _target_backend(dst)
|
|
194
|
+
copy_attrs(src.attrs, dst.attrs, target_backend=target_backend)
|
|
195
|
+
|
|
196
|
+
for key in src.keys():
|
|
197
|
+
obj = src[key]
|
|
198
|
+
|
|
199
|
+
if is_dataset(obj):
|
|
200
|
+
_copy_rows(obj, dst, key, indices)
|
|
201
|
+
continue
|
|
202
|
+
|
|
203
|
+
if not is_group(obj):
|
|
204
|
+
continue
|
|
205
|
+
|
|
206
|
+
enc = _decode_attr(obj.attrs.get("encoding-type", b""))
|
|
207
|
+
|
|
208
|
+
if enc == spec.CATEGORICAL or (enc is None and "codes" in obj):
|
|
209
|
+
gdst = dst.create_group(key)
|
|
210
|
+
copy_attrs(obj.attrs, gdst.attrs, target_backend=target_backend)
|
|
211
|
+
copy_tree(obj["categories"], gdst, "categories")
|
|
212
|
+
_copy_rows(obj["codes"], gdst, "codes", indices)
|
|
213
|
+
continue
|
|
214
|
+
|
|
215
|
+
if enc in spec.MASKED_TYPES or ("values" in obj and "mask" in obj):
|
|
216
|
+
gdst = dst.create_group(key)
|
|
217
|
+
copy_attrs(obj.attrs, gdst.attrs, target_backend=target_backend)
|
|
218
|
+
_copy_rows(obj["values"], gdst, "values", indices)
|
|
219
|
+
_copy_rows(obj["mask"], gdst, "mask", indices)
|
|
220
|
+
continue
|
|
221
|
+
|
|
222
|
+
# Not row-aligned (e.g. __categories) -- copy verbatim.
|
|
223
|
+
copy_tree(obj, dst, key)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def subset_dense_matrix(
|
|
227
|
+
src: Any,
|
|
228
|
+
dst_parent: Any,
|
|
229
|
+
name: str,
|
|
230
|
+
obs_idx: Optional[np.ndarray],
|
|
231
|
+
var_idx: Optional[np.ndarray],
|
|
232
|
+
*,
|
|
233
|
+
chunk_rows: int = 1024,
|
|
234
|
+
) -> None:
|
|
235
|
+
if src.ndim != 2:
|
|
236
|
+
copy_tree(src, dst_parent, name)
|
|
237
|
+
return
|
|
238
|
+
|
|
239
|
+
n_obs, n_var = src.shape
|
|
240
|
+
out_obs = len(obs_idx) if obs_idx is not None else n_obs
|
|
241
|
+
out_var = len(var_idx) if var_idx is not None else n_var
|
|
242
|
+
|
|
243
|
+
target_backend = _target_backend(dst_parent)
|
|
244
|
+
kw = dataset_create_kwargs(
|
|
245
|
+
src, target_backend=target_backend, dst_parent=dst_parent
|
|
246
|
+
)
|
|
247
|
+
kw = _clamp_chunks(kw, out_obs, out_var)
|
|
248
|
+
|
|
249
|
+
dst = create_dataset(
|
|
250
|
+
dst_parent,
|
|
251
|
+
name,
|
|
252
|
+
shape=(out_obs, out_var),
|
|
253
|
+
dtype=src.dtype,
|
|
254
|
+
**kw,
|
|
255
|
+
)
|
|
256
|
+
copy_attrs(src.attrs, dst.attrs, target_backend=_target_backend(dst_parent))
|
|
257
|
+
|
|
258
|
+
for out_start in range(0, out_obs, chunk_rows):
|
|
259
|
+
out_end = min(out_start + chunk_rows, out_obs)
|
|
260
|
+
|
|
261
|
+
if obs_idx is None:
|
|
262
|
+
block = src[out_start:out_end, :]
|
|
263
|
+
else:
|
|
264
|
+
rows = obs_idx[out_start:out_end]
|
|
265
|
+
block = src[rows, :]
|
|
266
|
+
|
|
267
|
+
if var_idx is not None:
|
|
268
|
+
block = block[:, var_idx]
|
|
269
|
+
|
|
270
|
+
dst[out_start:out_end, :] = block
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _minor_remap(keep: Optional[np.ndarray], size: int) -> Optional[np.ndarray]:
|
|
274
|
+
"""Build a lookup from old minor index to new, with -1 for dropped entries.
|
|
275
|
+
|
|
276
|
+
A dense lookup table costs one int32 per column of the source, which is
|
|
277
|
+
negligible beside the matrix itself and turns the remap into a single
|
|
278
|
+
vectorised gather rather than a per-entry dict lookup.
|
|
279
|
+
"""
|
|
280
|
+
if keep is None:
|
|
281
|
+
return None
|
|
282
|
+
remap = np.full(size, -1, dtype=np.int64)
|
|
283
|
+
remap[keep] = np.arange(len(keep), dtype=np.int64)
|
|
284
|
+
return remap
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def subset_sparse_matrix_group(
|
|
288
|
+
src: Any,
|
|
289
|
+
dst_parent: Any,
|
|
290
|
+
name: str,
|
|
291
|
+
obs_idx: Optional[np.ndarray],
|
|
292
|
+
var_idx: Optional[np.ndarray],
|
|
293
|
+
*,
|
|
294
|
+
chunk_major: int = 4096,
|
|
295
|
+
) -> None:
|
|
296
|
+
"""Subset a CSR/CSC matrix, streaming it a block of major axis at a time.
|
|
297
|
+
|
|
298
|
+
Only the slice of `data`/`indices` spanned by the current block is read, so
|
|
299
|
+
peak memory is set by `chunk_major` rather than by the matrix. The output
|
|
300
|
+
datasets are grown as each block is appended, since the final nnz is not
|
|
301
|
+
known until the pass completes.
|
|
302
|
+
"""
|
|
303
|
+
enc = _decode_attr(src.attrs.get("encoding-type", b""))
|
|
304
|
+
if enc not in spec.SPARSE_TYPES:
|
|
305
|
+
raise ValueError(f"Unsupported sparse encoding type: {enc}")
|
|
306
|
+
|
|
307
|
+
shape = src.attrs.get("shape", None)
|
|
308
|
+
if shape is None:
|
|
309
|
+
raise ValueError("Sparse matrix group missing 'shape' attribute.")
|
|
310
|
+
n_rows, n_cols = int(shape[0]), int(shape[1])
|
|
311
|
+
|
|
312
|
+
data_ds, indices_ds = src["data"], src["indices"]
|
|
313
|
+
indptr = np.asarray(src["indptr"][...], dtype=np.int64)
|
|
314
|
+
|
|
315
|
+
if enc == spec.CSR_MATRIX:
|
|
316
|
+
major_idx, minor_keep, n_minor = obs_idx, var_idx, n_cols
|
|
317
|
+
out_rows = len(obs_idx) if obs_idx is not None else n_rows
|
|
318
|
+
out_cols = len(var_idx) if var_idx is not None else n_cols
|
|
319
|
+
else:
|
|
320
|
+
major_idx, minor_keep, n_minor = var_idx, obs_idx, n_rows
|
|
321
|
+
out_rows = len(obs_idx) if obs_idx is not None else n_rows
|
|
322
|
+
out_cols = len(var_idx) if var_idx is not None else n_cols
|
|
323
|
+
|
|
324
|
+
n_major = n_rows if enc == spec.CSR_MATRIX else n_cols
|
|
325
|
+
majors = major_idx if major_idx is not None else np.arange(n_major, dtype=np.int64)
|
|
326
|
+
remap = _minor_remap(minor_keep, n_minor)
|
|
327
|
+
|
|
328
|
+
group = dst_parent.create_group(name)
|
|
329
|
+
copy_attrs(src.attrs, group.attrs, target_backend=_target_backend(dst_parent))
|
|
330
|
+
spec.set_encoding(group, enc)
|
|
331
|
+
set_shape_attr(group, (out_rows, out_cols))
|
|
332
|
+
|
|
333
|
+
out_data = _growable(group, "data", data_ds.dtype)
|
|
334
|
+
out_indices = _growable(group, "indices", np.int64)
|
|
335
|
+
out_indptr = [0]
|
|
336
|
+
nnz = 0
|
|
337
|
+
|
|
338
|
+
for block_start in range(0, len(majors), chunk_major):
|
|
339
|
+
block = majors[block_start : block_start + chunk_major]
|
|
340
|
+
# One contiguous read covers the whole block's entries.
|
|
341
|
+
lo, hi = int(indptr[block].min()), int(indptr[block + 1].max())
|
|
342
|
+
if hi > lo:
|
|
343
|
+
block_indices = np.asarray(indices_ds[lo:hi], dtype=np.int64)
|
|
344
|
+
block_data = np.asarray(data_ds[lo:hi])
|
|
345
|
+
else:
|
|
346
|
+
block_indices = np.empty(0, dtype=np.int64)
|
|
347
|
+
block_data = np.empty(0, dtype=data_ds.dtype)
|
|
348
|
+
|
|
349
|
+
kept_indices: List[np.ndarray] = []
|
|
350
|
+
kept_data: List[np.ndarray] = []
|
|
351
|
+
for m in block:
|
|
352
|
+
sl = slice(int(indptr[m]) - lo, int(indptr[m + 1]) - lo)
|
|
353
|
+
minor = block_indices[sl]
|
|
354
|
+
values = block_data[sl]
|
|
355
|
+
if remap is not None:
|
|
356
|
+
mapped = remap[minor]
|
|
357
|
+
keep = mapped >= 0
|
|
358
|
+
minor, values = mapped[keep], values[keep]
|
|
359
|
+
kept_indices.append(minor)
|
|
360
|
+
kept_data.append(values)
|
|
361
|
+
nnz += len(minor)
|
|
362
|
+
out_indptr.append(nnz)
|
|
363
|
+
|
|
364
|
+
if kept_indices:
|
|
365
|
+
_append(out_indices, np.concatenate(kept_indices))
|
|
366
|
+
_append(out_data, np.concatenate(kept_data))
|
|
367
|
+
|
|
368
|
+
create_dataset(
|
|
369
|
+
group, "indptr", data=np.asarray(out_indptr, dtype=np.int64)
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _growable(group: Any, name: str, dtype: Any) -> Any:
|
|
374
|
+
"""Create an empty 1-D dataset that can be extended as blocks arrive."""
|
|
375
|
+
if is_zarr_group(group):
|
|
376
|
+
return group.create_array(name, shape=(0,), dtype=dtype, chunks=(65536,))
|
|
377
|
+
return group.create_dataset(
|
|
378
|
+
name, shape=(0,), maxshape=(None,), dtype=dtype, chunks=(65536,)
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _append(ds: Any, values: np.ndarray) -> None:
|
|
383
|
+
"""Append a block to a growable 1-D dataset."""
|
|
384
|
+
if values.size == 0:
|
|
385
|
+
return
|
|
386
|
+
start = ds.shape[0]
|
|
387
|
+
ds.resize((start + values.size,))
|
|
388
|
+
ds[start:] = values
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def subset_matrix_entry(
|
|
392
|
+
obj: Any,
|
|
393
|
+
dst_parent: Any,
|
|
394
|
+
name: str,
|
|
395
|
+
obs_idx: Optional[np.ndarray],
|
|
396
|
+
var_idx: Optional[np.ndarray],
|
|
397
|
+
*,
|
|
398
|
+
chunk_rows: int,
|
|
399
|
+
entry_label: str,
|
|
400
|
+
) -> None:
|
|
401
|
+
if is_dataset(obj):
|
|
402
|
+
subset_dense_matrix(
|
|
403
|
+
obj, dst_parent, name, obs_idx, var_idx, chunk_rows=chunk_rows
|
|
404
|
+
)
|
|
405
|
+
return
|
|
406
|
+
|
|
407
|
+
if is_group(obj):
|
|
408
|
+
enc = _decode_attr(obj.attrs.get("encoding-type", b""))
|
|
409
|
+
if enc in spec.SPARSE_TYPES:
|
|
410
|
+
subset_sparse_matrix_group(obj, dst_parent, name, obs_idx, var_idx)
|
|
411
|
+
return
|
|
412
|
+
if enc == spec.DATAFRAME:
|
|
413
|
+
# obsm/varm may hold a dataframe; it is row-aligned like obs/var.
|
|
414
|
+
subset_axis_group(obj, dst_parent.create_group(name), obs_idx)
|
|
415
|
+
return
|
|
416
|
+
raise ValueError(f"Unsupported {entry_label} encoding type: {enc}")
|
|
417
|
+
|
|
418
|
+
raise ValueError(f"Unsupported {entry_label} object type")
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
HANDLED_KEYS = frozenset(
|
|
422
|
+
{"obs", "var", "X", "layers", "obsm", "varm", "obsp", "varp", "uns", "raw"}
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def subset_raw_group(
|
|
427
|
+
src_raw: Any,
|
|
428
|
+
dst: Any,
|
|
429
|
+
obs_idx: Optional[np.ndarray],
|
|
430
|
+
var_keep: Optional[Set[str]],
|
|
431
|
+
*,
|
|
432
|
+
chunk_rows: int,
|
|
433
|
+
console: Console,
|
|
434
|
+
) -> None:
|
|
435
|
+
"""Subset a `raw/` group, which carries its own var axis.
|
|
436
|
+
|
|
437
|
+
`raw` typically holds more genes than the main object, so its var names are
|
|
438
|
+
matched independently rather than reusing the outer var indices -- using
|
|
439
|
+
those would select the wrong columns entirely.
|
|
440
|
+
"""
|
|
441
|
+
raw_dst = dst.create_group("raw")
|
|
442
|
+
copy_attrs(src_raw.attrs, raw_dst.attrs, target_backend=_target_backend(dst))
|
|
443
|
+
spec.set_encoding(raw_dst, spec.RAW)
|
|
444
|
+
|
|
445
|
+
raw_var_idx: Optional[np.ndarray] = None
|
|
446
|
+
if var_keep is not None and "var" in src_raw:
|
|
447
|
+
raw_var_names, _ = resolve_index(src_raw["var"], "var")
|
|
448
|
+
raw_var_idx, missing = indices_from_name_set(raw_var_names, var_keep)
|
|
449
|
+
console.print(
|
|
450
|
+
f"[green]Selected {len(raw_var_idx)} raw/var "
|
|
451
|
+
f"(of {element_len(raw_var_names)})[/]"
|
|
452
|
+
)
|
|
453
|
+
if missing:
|
|
454
|
+
console.print(
|
|
455
|
+
f"[yellow]Warning: {len(missing)} var names not found in raw/var[/]"
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
if "var" in src_raw:
|
|
459
|
+
subset_axis_group(src_raw["var"], raw_dst.create_group("var"), raw_var_idx)
|
|
460
|
+
|
|
461
|
+
if "X" in src_raw:
|
|
462
|
+
subset_matrix_entry(
|
|
463
|
+
src_raw["X"],
|
|
464
|
+
raw_dst,
|
|
465
|
+
"X",
|
|
466
|
+
obs_idx,
|
|
467
|
+
raw_var_idx,
|
|
468
|
+
chunk_rows=chunk_rows,
|
|
469
|
+
entry_label="raw/X",
|
|
470
|
+
)
|
|
471
|
+
|
|
472
|
+
if "varm" in src_raw:
|
|
473
|
+
varm_dst = _ensure_group(raw_dst, "varm")
|
|
474
|
+
for key in src_raw["varm"].keys():
|
|
475
|
+
subset_matrix_entry(
|
|
476
|
+
src_raw["varm"][key],
|
|
477
|
+
varm_dst,
|
|
478
|
+
key,
|
|
479
|
+
raw_var_idx,
|
|
480
|
+
None,
|
|
481
|
+
chunk_rows=chunk_rows,
|
|
482
|
+
entry_label=f"raw/varm:{key}",
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
for key in src_raw.keys():
|
|
486
|
+
if key not in ("X", "var", "varm"):
|
|
487
|
+
copy_tree(src_raw[key], raw_dst, key)
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _select_axis(
|
|
491
|
+
src: Any,
|
|
492
|
+
axis: str,
|
|
493
|
+
name_file: Optional[Path],
|
|
494
|
+
query: Optional[str],
|
|
495
|
+
console: Console,
|
|
496
|
+
) -> Optional[np.ndarray]:
|
|
497
|
+
"""Resolve an axis selection from a name list or a query, or None for all."""
|
|
498
|
+
if name_file is None and query is None:
|
|
499
|
+
return None
|
|
500
|
+
|
|
501
|
+
if name_file is not None and query is not None:
|
|
502
|
+
raise ValueError(f"Give either --{axis} or --{axis}-query, not both.")
|
|
503
|
+
|
|
504
|
+
if query is not None:
|
|
505
|
+
console.print(f"[cyan]Evaluating {axis} query...[/]")
|
|
506
|
+
indices = select_indices(src, axis, query)
|
|
507
|
+
console.print(f"[green]Selected {len(indices)} {axis}[/]")
|
|
508
|
+
if len(indices) == 0:
|
|
509
|
+
raise ValueError(f"The {axis} query matched no rows.")
|
|
510
|
+
return indices
|
|
511
|
+
|
|
512
|
+
keep = _read_name_file(name_file)
|
|
513
|
+
console.print(f"[cyan]Found {len(keep)} {axis} names to keep[/]")
|
|
514
|
+
names_ds, _ = resolve_index(src[axis], axis)
|
|
515
|
+
indices, missing = indices_from_name_set(names_ds, keep)
|
|
516
|
+
if missing:
|
|
517
|
+
console.print(
|
|
518
|
+
f"[yellow]Warning: {len(missing)} {axis} names not found in file[/]"
|
|
519
|
+
)
|
|
520
|
+
console.print(
|
|
521
|
+
f"[green]Selected {len(indices)} {axis} (of {element_len(names_ds)})[/]"
|
|
522
|
+
)
|
|
523
|
+
return indices
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def subset_h5ad(
|
|
527
|
+
file: Path,
|
|
528
|
+
output: Optional[Path],
|
|
529
|
+
obs_file: Optional[Path],
|
|
530
|
+
var_file: Optional[Path],
|
|
531
|
+
*,
|
|
532
|
+
chunk_rows: int = 1024,
|
|
533
|
+
console: Console,
|
|
534
|
+
inplace: bool = False,
|
|
535
|
+
obs_query: Optional[str] = None,
|
|
536
|
+
var_query: Optional[str] = None,
|
|
537
|
+
obs_indices: Optional[np.ndarray] = None,
|
|
538
|
+
var_indices: Optional[np.ndarray] = None,
|
|
539
|
+
zarr_format: Optional[int] = None,
|
|
540
|
+
quiet: bool = False,
|
|
541
|
+
) -> None:
|
|
542
|
+
"""Write a copy of `file` narrowed to the selected obs and/or var.
|
|
543
|
+
|
|
544
|
+
Selection comes from a name file, a query, or indices computed by a caller
|
|
545
|
+
such as `split`. Exactly one source per axis.
|
|
546
|
+
"""
|
|
547
|
+
has_selection = any(
|
|
548
|
+
x is not None
|
|
549
|
+
for x in (obs_file, var_file, obs_query, var_query, obs_indices, var_indices)
|
|
550
|
+
)
|
|
551
|
+
if not has_selection:
|
|
552
|
+
raise ValueError("At least one of --obs or --var must be provided.")
|
|
553
|
+
|
|
554
|
+
if not inplace and output is None:
|
|
555
|
+
raise ValueError("Output file is required unless --inplace is specified.")
|
|
556
|
+
|
|
557
|
+
if inplace:
|
|
558
|
+
src_backend = detect_backend(file)
|
|
559
|
+
if src_backend == "zarr":
|
|
560
|
+
base_name = file.stem if file.suffix else file.name
|
|
561
|
+
tmp_path = file.with_name(f"{base_name}.subset-tmp.zarr")
|
|
562
|
+
else:
|
|
563
|
+
tmp_path = file.with_name(f"{file.name}.subset-tmp")
|
|
564
|
+
if tmp_path.exists():
|
|
565
|
+
raise FileExistsError(f"Temporary path already exists: {tmp_path}")
|
|
566
|
+
dst_path = tmp_path
|
|
567
|
+
else:
|
|
568
|
+
dst_path = output
|
|
569
|
+
|
|
570
|
+
if zarr_format is None and detect_backend(file) == "zarr":
|
|
571
|
+
with open_store(file, "r") as probe:
|
|
572
|
+
zarr_format = probe.zarr_format
|
|
573
|
+
|
|
574
|
+
# Rich allows only one live display per console, and the progress bar
|
|
575
|
+
# below is a second one. The spinner is therefore stopped explicitly once
|
|
576
|
+
# the selection is known, rather than left running around it.
|
|
577
|
+
status = console.status("[magenta]Opening files...[/]")
|
|
578
|
+
status.start()
|
|
579
|
+
try:
|
|
580
|
+
with open_store(file, "r") as src_store, open_store(
|
|
581
|
+
dst_path, "w", zarr_format=zarr_format
|
|
582
|
+
) as dst_store:
|
|
583
|
+
src = src_store.root
|
|
584
|
+
dst = dst_store.root
|
|
585
|
+
|
|
586
|
+
obs_idx = (
|
|
587
|
+
obs_indices
|
|
588
|
+
if obs_indices is not None
|
|
589
|
+
else _select_axis(src, "obs", obs_file, obs_query, console)
|
|
590
|
+
)
|
|
591
|
+
var_idx = (
|
|
592
|
+
var_indices
|
|
593
|
+
if var_indices is not None
|
|
594
|
+
else _select_axis(src, "var", var_file, var_query, console)
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
# raw/ has its own var axis, so it is matched by name rather than
|
|
598
|
+
# by reusing these indices.
|
|
599
|
+
var_keep: Optional[Set[str]] = None
|
|
600
|
+
if var_idx is not None and "var" in src:
|
|
601
|
+
var_keep = set(read_names(src, "var", var_idx))
|
|
602
|
+
|
|
603
|
+
status.stop()
|
|
604
|
+
|
|
605
|
+
tasks: List[str] = []
|
|
606
|
+
if "obs" in src:
|
|
607
|
+
tasks.append("obs")
|
|
608
|
+
if "var" in src:
|
|
609
|
+
tasks.append("var")
|
|
610
|
+
if "X" in src:
|
|
611
|
+
tasks.append("X")
|
|
612
|
+
if "layers" in src:
|
|
613
|
+
tasks.extend([f"layer:{k}" for k in src["layers"].keys()])
|
|
614
|
+
if "obsm" in src:
|
|
615
|
+
tasks.extend([f"obsm:{k}" for k in src["obsm"].keys()])
|
|
616
|
+
if "varm" in src:
|
|
617
|
+
tasks.extend([f"varm:{k}" for k in src["varm"].keys()])
|
|
618
|
+
if "obsp" in src:
|
|
619
|
+
tasks.extend([f"obsp:{k}" for k in src["obsp"].keys()])
|
|
620
|
+
if "varp" in src:
|
|
621
|
+
tasks.extend([f"varp:{k}" for k in src["varp"].keys()])
|
|
622
|
+
if "uns" in src:
|
|
623
|
+
tasks.append("uns")
|
|
624
|
+
# anndata writes a placeholder `raw` even when there is none, and
|
|
625
|
+
# on Zarr that placeholder is an array rather than a group.
|
|
626
|
+
if "raw" in src and is_group(src["raw"]):
|
|
627
|
+
tasks.append("raw")
|
|
628
|
+
elif "raw" in src:
|
|
629
|
+
tasks.append("copy:raw")
|
|
630
|
+
|
|
631
|
+
passthrough = [
|
|
632
|
+
k for k in src.keys() if k not in HANDLED_KEYS
|
|
633
|
+
]
|
|
634
|
+
if passthrough:
|
|
635
|
+
console.print(
|
|
636
|
+
"[yellow]Copying unrecognised top-level "
|
|
637
|
+
f"{'keys' if len(passthrough) > 1 else 'key'} verbatim: "
|
|
638
|
+
f"{', '.join(sorted(passthrough))}[/]"
|
|
639
|
+
)
|
|
640
|
+
tasks.extend(f"copy:{k}" for k in passthrough)
|
|
641
|
+
|
|
642
|
+
with Progress(
|
|
643
|
+
SpinnerColumn(finished_text="[green]✓[/]"),
|
|
644
|
+
TextColumn("[progress.description]{task.description}"),
|
|
645
|
+
console=console,
|
|
646
|
+
transient=False,
|
|
647
|
+
) as progress:
|
|
648
|
+
for task in tasks:
|
|
649
|
+
task_id = progress.add_task(
|
|
650
|
+
f"[cyan]Subsetting {task}...[/]", total=None
|
|
651
|
+
)
|
|
652
|
+
if task == "obs":
|
|
653
|
+
obs_dst = dst.create_group("obs")
|
|
654
|
+
subset_axis_group(src["obs"], obs_dst, obs_idx)
|
|
655
|
+
elif task == "var":
|
|
656
|
+
var_dst = dst.create_group("var")
|
|
657
|
+
subset_axis_group(src["var"], var_dst, var_idx)
|
|
658
|
+
elif task == "X":
|
|
659
|
+
X = src["X"]
|
|
660
|
+
if is_dataset(X):
|
|
661
|
+
subset_dense_matrix(
|
|
662
|
+
X, dst, "X", obs_idx, var_idx, chunk_rows=chunk_rows
|
|
663
|
+
)
|
|
664
|
+
elif is_group(X):
|
|
665
|
+
subset_sparse_matrix_group(X, dst, "X", obs_idx, var_idx)
|
|
666
|
+
else:
|
|
667
|
+
copy_tree(X, dst, "X")
|
|
668
|
+
elif task.startswith("layer:"):
|
|
669
|
+
key = task.split(":", 1)[1]
|
|
670
|
+
layer_src = src["layers"][key]
|
|
671
|
+
layers_dst = _ensure_group(dst, "layers")
|
|
672
|
+
subset_matrix_entry(
|
|
673
|
+
layer_src,
|
|
674
|
+
layers_dst,
|
|
675
|
+
key,
|
|
676
|
+
obs_idx,
|
|
677
|
+
var_idx,
|
|
678
|
+
chunk_rows=chunk_rows,
|
|
679
|
+
entry_label=f"layer:{key}",
|
|
680
|
+
)
|
|
681
|
+
elif task.startswith("obsm:"):
|
|
682
|
+
key = task.split(":", 1)[1]
|
|
683
|
+
obsm_dst = _ensure_group(dst, "obsm")
|
|
684
|
+
obsm_obj = src["obsm"][key]
|
|
685
|
+
subset_matrix_entry(
|
|
686
|
+
obsm_obj,
|
|
687
|
+
obsm_dst,
|
|
688
|
+
key,
|
|
689
|
+
obs_idx,
|
|
690
|
+
None,
|
|
691
|
+
chunk_rows=chunk_rows,
|
|
692
|
+
entry_label=f"obsm:{key}",
|
|
693
|
+
)
|
|
694
|
+
elif task.startswith("varm:"):
|
|
695
|
+
key = task.split(":", 1)[1]
|
|
696
|
+
varm_dst = _ensure_group(dst, "varm")
|
|
697
|
+
varm_obj = src["varm"][key]
|
|
698
|
+
subset_matrix_entry(
|
|
699
|
+
varm_obj,
|
|
700
|
+
varm_dst,
|
|
701
|
+
key,
|
|
702
|
+
var_idx,
|
|
703
|
+
None,
|
|
704
|
+
chunk_rows=chunk_rows,
|
|
705
|
+
entry_label=f"varm:{key}",
|
|
706
|
+
)
|
|
707
|
+
elif task.startswith("obsp:"):
|
|
708
|
+
key = task.split(":", 1)[1]
|
|
709
|
+
obsp_dst = _ensure_group(dst, "obsp")
|
|
710
|
+
obsp_obj = src["obsp"][key]
|
|
711
|
+
subset_matrix_entry(
|
|
712
|
+
obsp_obj,
|
|
713
|
+
obsp_dst,
|
|
714
|
+
key,
|
|
715
|
+
obs_idx,
|
|
716
|
+
obs_idx,
|
|
717
|
+
chunk_rows=chunk_rows,
|
|
718
|
+
entry_label=f"obsp:{key}",
|
|
719
|
+
)
|
|
720
|
+
elif task.startswith("varp:"):
|
|
721
|
+
key = task.split(":", 1)[1]
|
|
722
|
+
varp_dst = _ensure_group(dst, "varp")
|
|
723
|
+
varp_obj = src["varp"][key]
|
|
724
|
+
subset_matrix_entry(
|
|
725
|
+
varp_obj,
|
|
726
|
+
varp_dst,
|
|
727
|
+
key,
|
|
728
|
+
var_idx,
|
|
729
|
+
var_idx,
|
|
730
|
+
chunk_rows=chunk_rows,
|
|
731
|
+
entry_label=f"varp:{key}",
|
|
732
|
+
)
|
|
733
|
+
elif task == "uns":
|
|
734
|
+
copy_tree(src["uns"], dst, "uns")
|
|
735
|
+
elif task == "raw":
|
|
736
|
+
subset_raw_group(
|
|
737
|
+
src["raw"],
|
|
738
|
+
dst,
|
|
739
|
+
obs_idx,
|
|
740
|
+
var_keep,
|
|
741
|
+
chunk_rows=chunk_rows,
|
|
742
|
+
console=console,
|
|
743
|
+
)
|
|
744
|
+
elif task.startswith("copy:"):
|
|
745
|
+
key = task.split(":", 1)[1]
|
|
746
|
+
copy_tree(src[key], dst, key)
|
|
747
|
+
progress.update(
|
|
748
|
+
task_id,
|
|
749
|
+
description=f"[green]Subsetting {task}[/]",
|
|
750
|
+
completed=1,
|
|
751
|
+
total=1,
|
|
752
|
+
)
|
|
753
|
+
|
|
754
|
+
_ensure_optional_anndata_groups(dst)
|
|
755
|
+
finally:
|
|
756
|
+
status.stop()
|
|
757
|
+
|
|
758
|
+
if inplace:
|
|
759
|
+
if file.exists():
|
|
760
|
+
if file.is_dir():
|
|
761
|
+
shutil.rmtree(file)
|
|
762
|
+
else:
|
|
763
|
+
file.unlink()
|
|
764
|
+
if dst_path.is_dir():
|
|
765
|
+
shutil.move(str(dst_path), str(file))
|
|
766
|
+
else:
|
|
767
|
+
dst_path.replace(file)
|