pyadata-cli 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- adata/__init__.py +10 -0
- adata/cli.py +1139 -0
- adata/commands/__init__.py +8 -0
- adata/commands/concat.py +21 -0
- adata/commands/create.py +93 -0
- adata/commands/export.py +105 -0
- adata/commands/import_data.py +144 -0
- adata/commands/info.py +199 -0
- adata/commands/ls.py +114 -0
- adata/commands/split.py +152 -0
- adata/commands/subset.py +17 -0
- adata/core/__init__.py +1 -0
- adata/core/concat.py +1084 -0
- adata/core/info.py +276 -0
- adata/core/query.py +268 -0
- adata/core/read.py +28 -0
- adata/core/select.py +122 -0
- adata/core/subset.py +767 -0
- adata/elements/__init__.py +10 -0
- adata/elements/read.py +310 -0
- adata/elements/spec.py +125 -0
- adata/elements/strings.py +90 -0
- adata/elements/write.py +255 -0
- adata/formats/__init__.py +1 -0
- adata/formats/array.py +109 -0
- adata/formats/common.py +51 -0
- adata/formats/dataframe.py +234 -0
- adata/formats/image.py +85 -0
- adata/formats/json_data.py +196 -0
- adata/formats/sparse.py +253 -0
- adata/formats/validate.py +180 -0
- adata/info.py +3 -0
- adata/py.typed +0 -0
- adata/read.py +19 -0
- adata/storage/__init__.py +529 -0
- adata/util/__init__.py +1 -0
- adata/util/path.py +9 -0
- pyadata_cli-0.5.0.dist-info/METADATA +135 -0
- pyadata_cli-0.5.0.dist-info/RECORD +42 -0
- pyadata_cli-0.5.0.dist-info/WHEEL +4 -0
- pyadata_cli-0.5.0.dist-info/entry_points.txt +4 -0
- pyadata_cli-0.5.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,529 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any, Iterable, Optional, Sequence
|
|
6
|
+
import shutil
|
|
7
|
+
import warnings
|
|
8
|
+
|
|
9
|
+
import h5py
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
import zarr
|
|
13
|
+
except Exception: # pragma: no cover - optional dependency
|
|
14
|
+
zarr = None
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
ROOT_ENCODING_TYPE = "anndata"
|
|
20
|
+
ROOT_ENCODING_VERSION = "0.1.0"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class Store:
|
|
25
|
+
backend: str
|
|
26
|
+
root: Any
|
|
27
|
+
path: Path
|
|
28
|
+
zarr_format: Optional[int] = None
|
|
29
|
+
#: Whether to rewrite the Zarr consolidated metadata index on close.
|
|
30
|
+
consolidate: bool = False
|
|
31
|
+
|
|
32
|
+
def close(self) -> None:
|
|
33
|
+
if self.backend == "hdf5":
|
|
34
|
+
try:
|
|
35
|
+
self.root.close()
|
|
36
|
+
except Exception:
|
|
37
|
+
return
|
|
38
|
+
return
|
|
39
|
+
|
|
40
|
+
if self.consolidate and zarr is not None:
|
|
41
|
+
# anndata writes a consolidated metadata index at the root. New
|
|
42
|
+
# members land on disk regardless, but every reader that honours
|
|
43
|
+
# the index -- anndata included -- keeps using the stale copy and
|
|
44
|
+
# cannot see them, so the write looks like a silent no-op.
|
|
45
|
+
try:
|
|
46
|
+
with warnings.catch_warnings():
|
|
47
|
+
# zarr notes that consolidated metadata is not part of the
|
|
48
|
+
# v3 spec. We write it deliberately, because anndata does
|
|
49
|
+
# and because without it our writes are invisible to
|
|
50
|
+
# readers that trust the index.
|
|
51
|
+
warnings.simplefilter("ignore")
|
|
52
|
+
zarr.consolidate_metadata(self.root.store, path=self.root.path)
|
|
53
|
+
except Exception:
|
|
54
|
+
return
|
|
55
|
+
|
|
56
|
+
def __enter__(self) -> "Store":
|
|
57
|
+
return self
|
|
58
|
+
|
|
59
|
+
def __exit__(self, exc_type, exc, tb) -> None:
|
|
60
|
+
self.close()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _require_zarr() -> None:
|
|
64
|
+
if zarr is None: # pragma: no cover - optional dependency
|
|
65
|
+
raise ImportError(
|
|
66
|
+
"zarr is required for .zarr support. Install with: uv sync --extra zarr"
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def is_hdf5_group(obj: Any) -> bool:
|
|
71
|
+
return isinstance(obj, (h5py.File, h5py.Group))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def is_hdf5_dataset(obj: Any) -> bool:
|
|
75
|
+
return isinstance(obj, h5py.Dataset)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def is_zarr_group(obj: Any) -> bool:
|
|
79
|
+
return zarr is not None and isinstance(obj, zarr.Group)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def is_zarr_array(obj: Any) -> bool:
|
|
83
|
+
return zarr is not None and isinstance(obj, zarr.Array)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def is_group(obj: Any) -> bool:
|
|
87
|
+
return is_hdf5_group(obj) or is_zarr_group(obj)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def is_dataset(obj: Any) -> bool:
|
|
91
|
+
return is_hdf5_dataset(obj) or is_zarr_array(obj)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def is_zarr_path(path: Path) -> bool:
|
|
95
|
+
if not path.exists() or not path.is_dir():
|
|
96
|
+
return False
|
|
97
|
+
if (path / "zarr.json").exists():
|
|
98
|
+
return True
|
|
99
|
+
if (path / ".zgroup").exists() or (path / ".zattrs").exists():
|
|
100
|
+
return True
|
|
101
|
+
return False
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def detect_backend(path: Path) -> str:
|
|
105
|
+
if path.exists():
|
|
106
|
+
if path.is_dir():
|
|
107
|
+
if is_zarr_path(path):
|
|
108
|
+
return "zarr"
|
|
109
|
+
raise ValueError(
|
|
110
|
+
f"Path '{path}' is a directory but does not look like a Zarr store."
|
|
111
|
+
)
|
|
112
|
+
return "hdf5"
|
|
113
|
+
if path.suffix == ".zarr":
|
|
114
|
+
return "zarr"
|
|
115
|
+
return "hdf5"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def open_store(
|
|
119
|
+
path: Path,
|
|
120
|
+
mode: str,
|
|
121
|
+
zarr_format: Optional[int] = None,
|
|
122
|
+
require_anndata: bool = True,
|
|
123
|
+
) -> Store:
|
|
124
|
+
"""Open a store, auto-detecting the backend from `path`.
|
|
125
|
+
|
|
126
|
+
`zarr_format` selects the Zarr spec version for a store being created;
|
|
127
|
+
without it zarr-python uses its own default, which is v3. Callers writing a
|
|
128
|
+
derived store should pass the source store's format so a v2 input is not
|
|
129
|
+
silently upgraded.
|
|
130
|
+
|
|
131
|
+
Set `require_anndata=False` for format-agnostic commands such as `ls`,
|
|
132
|
+
which are expected to open plain HDF5 and Zarr stores and should not warn
|
|
133
|
+
about a missing AnnData root.
|
|
134
|
+
"""
|
|
135
|
+
path = Path(path)
|
|
136
|
+
backend = detect_backend(path)
|
|
137
|
+
if backend == "zarr":
|
|
138
|
+
_require_zarr()
|
|
139
|
+
kwargs: dict = {}
|
|
140
|
+
if zarr_format is not None:
|
|
141
|
+
kwargs["zarr_format"] = zarr_format
|
|
142
|
+
|
|
143
|
+
writable = _is_writable_mode(mode)
|
|
144
|
+
if writable:
|
|
145
|
+
# Work against the real hierarchy: a consolidated index is a
|
|
146
|
+
# snapshot, so members added through it are invisible even to the
|
|
147
|
+
# handle that created them.
|
|
148
|
+
kwargs["use_consolidated"] = False
|
|
149
|
+
|
|
150
|
+
root = zarr.open_group(str(path), mode=mode, **kwargs)
|
|
151
|
+
if writable:
|
|
152
|
+
ensure_anndata_root_attrs(root)
|
|
153
|
+
elif require_anndata:
|
|
154
|
+
warn_if_missing_anndata_root_attrs(root, path=path)
|
|
155
|
+
return Store(
|
|
156
|
+
backend="zarr",
|
|
157
|
+
root=root,
|
|
158
|
+
path=path,
|
|
159
|
+
zarr_format=zarr_format_of(root),
|
|
160
|
+
consolidate=writable,
|
|
161
|
+
)
|
|
162
|
+
root = h5py.File(path, mode)
|
|
163
|
+
if _is_writable_mode(mode):
|
|
164
|
+
ensure_anndata_root_attrs(root)
|
|
165
|
+
elif require_anndata:
|
|
166
|
+
warn_if_missing_anndata_root_attrs(root, path=path)
|
|
167
|
+
return Store(backend="hdf5", root=root, path=path)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def zarr_format_of(obj: Any) -> Optional[int]:
|
|
171
|
+
"""The Zarr spec version (2 or 3) backing `obj`, or None for HDF5."""
|
|
172
|
+
return getattr(getattr(obj, "metadata", None), "zarr_format", None)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _decode_attr(value: Any) -> Any:
|
|
176
|
+
if isinstance(value, bytes):
|
|
177
|
+
return value.decode("utf-8")
|
|
178
|
+
return value
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _is_writable_mode(mode: str) -> bool:
|
|
182
|
+
return any(flag in mode for flag in ("w", "a", "+", "x"))
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def has_valid_anndata_root_attrs(root: Any) -> bool:
|
|
186
|
+
enc_type = _decode_attr(root.attrs.get("encoding-type", None))
|
|
187
|
+
enc_ver = _decode_attr(root.attrs.get("encoding-version", None))
|
|
188
|
+
return enc_type == ROOT_ENCODING_TYPE and enc_ver == ROOT_ENCODING_VERSION
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def ensure_anndata_root_attrs(root: Any) -> None:
|
|
192
|
+
root.attrs["encoding-type"] = ROOT_ENCODING_TYPE
|
|
193
|
+
root.attrs["encoding-version"] = ROOT_ENCODING_VERSION
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def warn_if_missing_anndata_root_attrs(root: Any, *, path: Path) -> None:
|
|
197
|
+
if has_valid_anndata_root_attrs(root):
|
|
198
|
+
return
|
|
199
|
+
|
|
200
|
+
enc_type = _decode_attr(root.attrs.get("encoding-type", None))
|
|
201
|
+
enc_ver = _decode_attr(root.attrs.get("encoding-version", None))
|
|
202
|
+
warnings.warn(
|
|
203
|
+
(
|
|
204
|
+
f"Store '{path}' root has missing or invalid AnnData attrs "
|
|
205
|
+
f"(encoding-type={ROOT_ENCODING_TYPE!r}, encoding-version={ROOT_ENCODING_VERSION!r}). "
|
|
206
|
+
f"Found encoding-type={enc_type!r}, encoding-version={enc_ver!r}."
|
|
207
|
+
),
|
|
208
|
+
UserWarning,
|
|
209
|
+
stacklevel=2,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _normalize_attr_value(value: Any, target_backend: str) -> Any:
|
|
214
|
+
if target_backend == "zarr":
|
|
215
|
+
if isinstance(value, bytes):
|
|
216
|
+
return value.decode("utf-8")
|
|
217
|
+
if isinstance(value, (list, tuple)):
|
|
218
|
+
return [v.decode("utf-8") if isinstance(v, bytes) else v for v in value]
|
|
219
|
+
if isinstance(value, np.ndarray):
|
|
220
|
+
if value.dtype.kind in ("S", "O"):
|
|
221
|
+
return [
|
|
222
|
+
v.decode("utf-8") if isinstance(v, bytes) else v
|
|
223
|
+
for v in value.tolist()
|
|
224
|
+
]
|
|
225
|
+
return value.tolist()
|
|
226
|
+
if isinstance(value, np.generic):
|
|
227
|
+
return value.item()
|
|
228
|
+
return value
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def copy_attrs(src_attrs: Any, dst_attrs: Any, *, target_backend: str) -> None:
|
|
232
|
+
"""Copy attributes across, normalising values for the target backend.
|
|
233
|
+
|
|
234
|
+
Written in one go on Zarr. Each `attrs[k] = v` there persists the whole
|
|
235
|
+
metadata document through zarr's sync-over-async bridge, so setting them
|
|
236
|
+
one at a time means a separate round trip per attribute -- slow, and the
|
|
237
|
+
place a `split` over many groups was observed to wedge in CI.
|
|
238
|
+
"""
|
|
239
|
+
normalized = {
|
|
240
|
+
str(k): _normalize_attr_value(v, target_backend)
|
|
241
|
+
for k, v in src_attrs.items()
|
|
242
|
+
}
|
|
243
|
+
if not normalized:
|
|
244
|
+
return
|
|
245
|
+
|
|
246
|
+
put = getattr(dst_attrs, "put", None)
|
|
247
|
+
if target_backend == "zarr" and callable(put):
|
|
248
|
+
put({**dict(dst_attrs), **normalized})
|
|
249
|
+
return
|
|
250
|
+
|
|
251
|
+
for k, v in normalized.items():
|
|
252
|
+
dst_attrs[k] = v
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def dataset_create_kwargs(
|
|
256
|
+
src: Any,
|
|
257
|
+
*,
|
|
258
|
+
target_backend: str,
|
|
259
|
+
zarr_format: Optional[int] = None,
|
|
260
|
+
dst_parent: Any = None,
|
|
261
|
+
) -> dict:
|
|
262
|
+
"""Derive creation kwargs that carry a source's layout onto a new dataset.
|
|
263
|
+
|
|
264
|
+
Chunking, compression and sharding are preserved where the target backend
|
|
265
|
+
can express them; codecs that do not survive the crossing are dropped
|
|
266
|
+
rather than forwarded into an error.
|
|
267
|
+
"""
|
|
268
|
+
# Prefer the destination's own version over anything the caller guessed,
|
|
269
|
+
# so a call site that forgets to pass one still behaves correctly.
|
|
270
|
+
kw_target = zarr_format
|
|
271
|
+
if dst_parent is not None:
|
|
272
|
+
kw_target = zarr_format_of(dst_parent)
|
|
273
|
+
kw: dict = {}
|
|
274
|
+
chunks = getattr(src, "chunks", None)
|
|
275
|
+
if chunks is not None:
|
|
276
|
+
kw["chunks"] = chunks
|
|
277
|
+
if target_backend == "hdf5" and is_hdf5_dataset(src):
|
|
278
|
+
if src.compression is not None:
|
|
279
|
+
kw["compression"] = src.compression
|
|
280
|
+
kw["compression_opts"] = src.compression_opts
|
|
281
|
+
kw["shuffle"] = bool(src.shuffle)
|
|
282
|
+
kw["fletcher32"] = bool(src.fletcher32)
|
|
283
|
+
if src.scaleoffset is not None:
|
|
284
|
+
kw["scaleoffset"] = src.scaleoffset
|
|
285
|
+
if src.fillvalue is not None:
|
|
286
|
+
kw["fillvalue"] = src.fillvalue
|
|
287
|
+
if target_backend == "zarr" and is_zarr_array(src):
|
|
288
|
+
src_zarr_format = getattr(getattr(src, "metadata", None), "zarr_format", None)
|
|
289
|
+
target_format = _target_zarr_format(kw_target)
|
|
290
|
+
# Only when both versions are known and equal can codecs travel.
|
|
291
|
+
same_version = (
|
|
292
|
+
target_format is not None and src_zarr_format == target_format
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
# Codecs only travel between stores of the same Zarr version: v2 holds
|
|
296
|
+
# numcodecs objects, v3 holds its own codec classes, and neither
|
|
297
|
+
# accepts the other's. Across versions the target's default is used
|
|
298
|
+
# rather than a translation that fails at creation time.
|
|
299
|
+
if same_version:
|
|
300
|
+
if src_zarr_format == 3:
|
|
301
|
+
try:
|
|
302
|
+
compressors = getattr(src, "compressors", None)
|
|
303
|
+
except Exception:
|
|
304
|
+
compressors = None
|
|
305
|
+
if compressors is not None:
|
|
306
|
+
kw["compressors"] = compressors
|
|
307
|
+
else:
|
|
308
|
+
try:
|
|
309
|
+
compressor = getattr(src, "compressor", None)
|
|
310
|
+
except Exception:
|
|
311
|
+
compressor = None
|
|
312
|
+
if compressor is not None:
|
|
313
|
+
kw["compressor"] = compressor
|
|
314
|
+
|
|
315
|
+
try:
|
|
316
|
+
filters = getattr(src, "filters", None)
|
|
317
|
+
except Exception:
|
|
318
|
+
filters = None
|
|
319
|
+
# A v2 string array carries VLenUTF8 in `filters`; the v3 string
|
|
320
|
+
# dtype encodes variable length itself and rejects it.
|
|
321
|
+
if filters and not _is_string_src(src):
|
|
322
|
+
kw["filters"] = filters
|
|
323
|
+
|
|
324
|
+
try:
|
|
325
|
+
shards = getattr(src, "shards", None)
|
|
326
|
+
except Exception:
|
|
327
|
+
shards = None
|
|
328
|
+
if shards is not None and target_format == 3:
|
|
329
|
+
kw["shards"] = shards
|
|
330
|
+
|
|
331
|
+
try:
|
|
332
|
+
fill_value = getattr(src, "fill_value", None)
|
|
333
|
+
except Exception:
|
|
334
|
+
fill_value = None
|
|
335
|
+
if fill_value is not None and not _is_string_src(src):
|
|
336
|
+
kw["fill_value"] = fill_value
|
|
337
|
+
return kw
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _create_string_dataset(
|
|
341
|
+
parent: Any,
|
|
342
|
+
name: str,
|
|
343
|
+
data: Any,
|
|
344
|
+
**kwargs: Any,
|
|
345
|
+
) -> Any:
|
|
346
|
+
"""Create a spec-compliant variable-length UTF-8 array from `data`.
|
|
347
|
+
|
|
348
|
+
Text needs a different spelling on each backend and neither accepts the
|
|
349
|
+
other's: Zarr rejects the `object` dtype an h5py vlen dataset reports, and
|
|
350
|
+
h5py rejects Zarr's `<U` dtype. Both are routed to variable-length UTF-8
|
|
351
|
+
here, which is what `string-array` requires.
|
|
352
|
+
"""
|
|
353
|
+
from adata.elements.strings import as_str_array, string_dtype_for
|
|
354
|
+
|
|
355
|
+
values = as_str_array(data)
|
|
356
|
+
|
|
357
|
+
if is_zarr_group(parent):
|
|
358
|
+
kwargs.pop("compressor", None)
|
|
359
|
+
kwargs.pop("filters", None)
|
|
360
|
+
arr = parent.create_array(
|
|
361
|
+
name,
|
|
362
|
+
shape=values.shape,
|
|
363
|
+
dtype=string_dtype_for("zarr"),
|
|
364
|
+
**kwargs,
|
|
365
|
+
)
|
|
366
|
+
if values.shape == ():
|
|
367
|
+
arr[()] = values.item()
|
|
368
|
+
else:
|
|
369
|
+
arr[...] = values
|
|
370
|
+
return arr
|
|
371
|
+
|
|
372
|
+
return parent.create_dataset(
|
|
373
|
+
name, data=values, dtype=string_dtype_for("hdf5"), **kwargs
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def create_dataset(
|
|
378
|
+
parent: Any,
|
|
379
|
+
name: str,
|
|
380
|
+
*,
|
|
381
|
+
data: Any = None,
|
|
382
|
+
shape: Optional[Sequence[int]] = None,
|
|
383
|
+
dtype: Any = None,
|
|
384
|
+
**kwargs: Any,
|
|
385
|
+
) -> Any:
|
|
386
|
+
"""Create an array under `parent`, backend-agnostically.
|
|
387
|
+
|
|
388
|
+
String data is always written as variable-length UTF-8 regardless of how
|
|
389
|
+
it arrived, so callers can hand over bytes, `<U`, `object` or StringDType
|
|
390
|
+
without knowing what the destination backend needs.
|
|
391
|
+
"""
|
|
392
|
+
from adata.elements.strings import is_string_dtype
|
|
393
|
+
|
|
394
|
+
if data is not None and dtype is None:
|
|
395
|
+
probe = data if hasattr(data, "dtype") else np.asarray(data)
|
|
396
|
+
if is_string_dtype(getattr(probe, "dtype", None)):
|
|
397
|
+
return _create_string_dataset(parent, name, probe, **kwargs)
|
|
398
|
+
elif data is None and dtype is not None and is_string_dtype(dtype):
|
|
399
|
+
from adata.elements.strings import string_dtype_for
|
|
400
|
+
|
|
401
|
+
backend = "zarr" if is_zarr_group(parent) else "hdf5"
|
|
402
|
+
dtype = string_dtype_for(backend)
|
|
403
|
+
if is_zarr_group(parent):
|
|
404
|
+
kwargs.pop("compressor", None)
|
|
405
|
+
kwargs.pop("filters", None)
|
|
406
|
+
|
|
407
|
+
if is_zarr_group(parent):
|
|
408
|
+
zarr_format = getattr(getattr(parent, "metadata", None), "zarr_format", None)
|
|
409
|
+
if zarr_format == 3:
|
|
410
|
+
kwargs = dict(kwargs)
|
|
411
|
+
kwargs.pop("compressor", None)
|
|
412
|
+
elif (
|
|
413
|
+
zarr_format == 2 and "compressors" in kwargs and "compressor" not in kwargs
|
|
414
|
+
):
|
|
415
|
+
kwargs = dict(kwargs)
|
|
416
|
+
compressors = kwargs.pop("compressors")
|
|
417
|
+
if isinstance(compressors, (list, tuple)) and len(compressors) == 1:
|
|
418
|
+
kwargs["compressor"] = compressors[0]
|
|
419
|
+
if data is not None:
|
|
420
|
+
return parent.create_array(name, data=data, **kwargs)
|
|
421
|
+
return parent.create_array(name, shape=shape, dtype=dtype, **kwargs)
|
|
422
|
+
if data is not None:
|
|
423
|
+
return parent.create_dataset(name, data=data, **kwargs)
|
|
424
|
+
return parent.create_dataset(name, shape=shape, dtype=dtype, **kwargs)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _target_zarr_format(zarr_format: Optional[int]) -> Optional[int]:
|
|
428
|
+
"""The destination's Zarr version, or None when the caller did not say.
|
|
429
|
+
|
|
430
|
+
Deliberately not defaulting to 3: guessing meant v3-only options such as
|
|
431
|
+
sharding were forwarded into v2 arrays, which reject them outright.
|
|
432
|
+
Unknown means "carry nothing version-specific".
|
|
433
|
+
"""
|
|
434
|
+
return zarr_format
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _is_string_src(src: Any) -> bool:
|
|
438
|
+
from adata.elements.strings import is_string_dtype
|
|
439
|
+
|
|
440
|
+
return is_string_dtype(getattr(src, "dtype", None))
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _chunk_step(shape: Sequence[int], chunks: Optional[Sequence[int]]) -> int:
|
|
444
|
+
if chunks is not None and len(chunks) > 0 and chunks[0]:
|
|
445
|
+
return int(chunks[0])
|
|
446
|
+
if not shape:
|
|
447
|
+
return 1
|
|
448
|
+
return max(1, min(1024, int(shape[0])))
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def copy_dataset(src: Any, dst_group: Any, name: str) -> Any:
|
|
452
|
+
"""Copy a dataset into `dst_group`, streaming it in chunks.
|
|
453
|
+
|
|
454
|
+
The destination dtype is resolved for the target backend rather than
|
|
455
|
+
reused: an h5py variable-length string dataset reports `object`, which Zarr
|
|
456
|
+
refuses to create, so copying one verbatim fails on every real store.
|
|
457
|
+
"""
|
|
458
|
+
from adata.elements.strings import target_dtype
|
|
459
|
+
|
|
460
|
+
shape = tuple(src.shape) if getattr(src, "shape", None) is not None else ()
|
|
461
|
+
target_backend = "zarr" if is_zarr_group(dst_group) else "hdf5"
|
|
462
|
+
zformat = zarr_format_of(dst_group)
|
|
463
|
+
ds = create_dataset(
|
|
464
|
+
dst_group,
|
|
465
|
+
name,
|
|
466
|
+
shape=shape,
|
|
467
|
+
dtype=target_dtype(src.dtype, target_backend, zformat),
|
|
468
|
+
**dataset_create_kwargs(
|
|
469
|
+
src, target_backend=target_backend, zarr_format=zformat
|
|
470
|
+
),
|
|
471
|
+
)
|
|
472
|
+
copy_attrs(src.attrs, ds.attrs, target_backend=target_backend)
|
|
473
|
+
|
|
474
|
+
if shape == ():
|
|
475
|
+
ds[()] = src[()]
|
|
476
|
+
return ds
|
|
477
|
+
|
|
478
|
+
step = _chunk_step(shape, getattr(src, "chunks", None))
|
|
479
|
+
for start in range(0, shape[0], step):
|
|
480
|
+
end = min(start + step, shape[0])
|
|
481
|
+
if len(shape) == 1:
|
|
482
|
+
ds[start:end] = src[start:end]
|
|
483
|
+
else:
|
|
484
|
+
ds[start:end, ...] = src[start:end, ...]
|
|
485
|
+
return ds
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def copy_tree(
|
|
489
|
+
src_obj: Any, dst_group: Any, name: str, *, exclude: Iterable[str] = ()
|
|
490
|
+
) -> Any:
|
|
491
|
+
if is_hdf5_group(dst_group) and (
|
|
492
|
+
is_hdf5_group(src_obj) or is_hdf5_dataset(src_obj)
|
|
493
|
+
):
|
|
494
|
+
if not exclude:
|
|
495
|
+
dst_group.copy(src_obj, dst_group, name)
|
|
496
|
+
return dst_group[name]
|
|
497
|
+
if is_dataset(src_obj):
|
|
498
|
+
return copy_dataset(src_obj, dst_group, name)
|
|
499
|
+
if not is_group(src_obj):
|
|
500
|
+
raise TypeError(f"Unsupported object type for copy: {type(src_obj)}")
|
|
501
|
+
|
|
502
|
+
target_backend = "zarr" if is_zarr_group(dst_group) else "hdf5"
|
|
503
|
+
grp = dst_group.create_group(name)
|
|
504
|
+
copy_attrs(src_obj.attrs, grp.attrs, target_backend=target_backend)
|
|
505
|
+
for key in src_obj.keys():
|
|
506
|
+
if key in exclude:
|
|
507
|
+
continue
|
|
508
|
+
child = src_obj[key]
|
|
509
|
+
copy_tree(child, grp, key, exclude=exclude)
|
|
510
|
+
return grp
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def copy_store_contents(src_root: Any, dst_root: Any) -> None:
|
|
514
|
+
target_backend = "zarr" if is_zarr_group(dst_root) else "hdf5"
|
|
515
|
+
copy_attrs(src_root.attrs, dst_root.attrs, target_backend=target_backend)
|
|
516
|
+
ensure_anndata_root_attrs(dst_root)
|
|
517
|
+
for key in src_root.keys():
|
|
518
|
+
copy_tree(src_root[key], dst_root, key)
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def copy_path(src: Path, dst: Path) -> None:
|
|
522
|
+
src = Path(src)
|
|
523
|
+
dst = Path(dst)
|
|
524
|
+
if is_zarr_path(src):
|
|
525
|
+
if dst.exists():
|
|
526
|
+
raise FileExistsError(f"Destination '{dst}' already exists.")
|
|
527
|
+
shutil.copytree(src, dst)
|
|
528
|
+
return
|
|
529
|
+
shutil.copy2(src, dst)
|
adata/util/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Utility helpers used across h5ad modules."""
|
adata/util/path.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pyadata-cli
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores
|
|
5
|
+
Keywords: anndata,h5ad,zarr,bioinformatics,single-cell,cli
|
|
6
|
+
Author: Aljes Binkevich
|
|
7
|
+
Author-email: Aljes Binkevich <ab76@sanger.ac.uk>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Dist: h5py>=3.15.1
|
|
21
|
+
Requires-Dist: numpy>=2.3.5
|
|
22
|
+
Requires-Dist: pillow>=12.1.0
|
|
23
|
+
Requires-Dist: rich>=14.2.0
|
|
24
|
+
Requires-Dist: typer>=0.20.0
|
|
25
|
+
Requires-Dist: zarr>=3.1.5
|
|
26
|
+
Requires-Dist: pytest>=8.3.4 ; extra == 'dev'
|
|
27
|
+
Requires-Dist: pytest-cov>=6.0.0 ; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest-timeout>=2.3.0 ; extra == 'dev'
|
|
29
|
+
Requires-Dist: anndata>=0.13 ; extra == 'dev'
|
|
30
|
+
Maintainer: Aljes Binkevich
|
|
31
|
+
Maintainer-email: Aljes Binkevich <ab76@sanger.ac.uk>
|
|
32
|
+
Requires-Python: >=3.12
|
|
33
|
+
Project-URL: Changelog, https://github.com/cellgeni/adata-cli/blob/main/CHANGELOG.md
|
|
34
|
+
Project-URL: Documentation, https://cellgeni.github.io/adata-cli/
|
|
35
|
+
Project-URL: Homepage, https://github.com/cellgeni/adata-cli
|
|
36
|
+
Project-URL: Issues, https://github.com/cellgeni/adata-cli/issues
|
|
37
|
+
Project-URL: Repository, https://github.com/cellgeni/adata-cli
|
|
38
|
+
Provides-Extra: dev
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
|
|
41
|
+
# adata CLI
|
|
42
|
+
|
|
43
|
+
A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) without loading them fully into memory. Streams data directly from disk for efficient inspection of structure, metadata, and matrices.
|
|
44
|
+
|
|
45
|
+
## Features
|
|
46
|
+
|
|
47
|
+
- Streaming access to very large `.h5ad` and `.zarr` stores
|
|
48
|
+
- Auto-detects `.h5ad` files vs `.zarr` directories
|
|
49
|
+
- Chunked processing for dense and sparse matrices (CSR/CSC)
|
|
50
|
+
- Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
|
|
51
|
+
- Converts between HDF5 and Zarr (v2 and v3) in either direction
|
|
52
|
+
- Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
|
|
53
|
+
|
|
54
|
+
**Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
|
|
55
|
+
|
|
56
|
+
## Installation
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
pip install pyadata-cli
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
The command is `adata`. The distribution is named `pyadata-cli` because
|
|
63
|
+
`adata-cli` was already taken on PyPI by an unrelated project.
|
|
64
|
+
|
|
65
|
+
From source with [uv](https://docs.astral.sh/uv/):
|
|
66
|
+
```bash
|
|
67
|
+
git clone https://github.com/cellgeni/adata-cli.git
|
|
68
|
+
cd adata-cli
|
|
69
|
+
uv sync
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
For development and testing:
|
|
73
|
+
```bash
|
|
74
|
+
uv sync --extra dev
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Alternative with pip:
|
|
78
|
+
```bash
|
|
79
|
+
git clone https://github.com/cellgeni/adata-cli.git
|
|
80
|
+
cd adata-cli
|
|
81
|
+
pip install .
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
For development and testing with pip:
|
|
85
|
+
```bash
|
|
86
|
+
pip install -e ".[dev]"
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Commands (Overview)
|
|
90
|
+
|
|
91
|
+
Run help at any level (e.g. `adata --help`, `adata export --help`).
|
|
92
|
+
|
|
93
|
+
- `view` – AnnData-aware inspection: store layout, shapes, and encodings; supports drilling into paths like `obsm/X_pca` or `uns`.
|
|
94
|
+
- `ls` – list the contents of any HDF5 or Zarr store as a tree, with no AnnData assumptions (works on `.loom` and plain `.h5`); `-1` emits bare paths for piping.
|
|
95
|
+
- `create` – write a new, empty AnnData store for `import` to fill in.
|
|
96
|
+
- `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
|
|
97
|
+
- `split` – write one store per distinct value of an annotation column, with a CSV manifest.
|
|
98
|
+
- `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
|
|
99
|
+
- `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
|
|
100
|
+
- `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
|
|
101
|
+
|
|
102
|
+
### Building a store from scratch
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
adata create out.h5ad --obs-names cells.txt --var-names genes.txt
|
|
106
|
+
adata import sparse out.h5ad X counts.mtx --inplace
|
|
107
|
+
adata import dataframe out.h5ad obs cells.csv --inplace -i cell_id
|
|
108
|
+
adata import array out.h5ad obsm/X_umap umap.npy --inplace
|
|
109
|
+
adata import dict out.h5ad uns/params params.json --inplace
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### Filtering without a name list
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
adata subset data.h5ad -o cortex.h5ad --obs-query "cluster == Cortex_2"
|
|
116
|
+
adata subset data.h5ad -o big.h5ad -q "n_counts > 1000 and cluster in A,B"
|
|
117
|
+
adata split data.h5ad --by sample -o per_sample/
|
|
118
|
+
adata concat per_sample/*.h5ad -o merged.h5ad --join outer --label sample
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
## Documentation
|
|
122
|
+
|
|
123
|
+
- [Get started](docs/GET_STARTED.md) — a short tutorial
|
|
124
|
+
- [Command reference](docs/COMMANDS.md) — every command and flag
|
|
125
|
+
- [Element spec: HDF5](docs/ELEMENTS_h5ad.md) / [Zarr](docs/ELEMENTS_zarr.md) — the on-disk format, and what this tool does with it
|
|
126
|
+
- [Testing](docs/TESTING.md) — how the suite is organised, and how compatibility is verified against six anndata releases
|
|
127
|
+
- [Changelog](CHANGELOG.md)
|
|
128
|
+
|
|
129
|
+
## Docker
|
|
130
|
+
|
|
131
|
+
A docker image is available on QUAY: `quay.io/cellgeni/adata-cli:latest`. Pull and run with:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
docker run --rm -it -v /path/to/data:/data quay.io/cellgeni/adata-cli:latest view /data/your_file.h5ad
|
|
135
|
+
```
|