pyadata-cli 0.7.0__tar.gz → 0.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/PKG-INFO +1 -1
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/pyproject.toml +1 -1
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/cli.py +23 -13
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/convert.py +2 -8
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/recast.py +52 -73
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/split.py +29 -14
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/concat.py +21 -1
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/subset.py +27 -34
- pyadata_cli-0.7.1/src/adata/util/path.py +19 -0
- pyadata_cli-0.7.1/src/adata/util/staging.py +99 -0
- pyadata_cli-0.7.0/src/adata/util/path.py +0 -9
- pyadata_cli-0.7.0/src/adata/util/staging.py +0 -63
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/LICENSE +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/README.md +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/concat.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/create.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/export.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/import_data.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/info.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/ls.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/subset.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/conversion/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/conversion/anndata_copy.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/conversion/base.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/conversion/tenx.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/info.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/query.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/read.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/recast.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/select.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/elements/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/elements/read.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/elements/spec.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/elements/strings.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/elements/write.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/array.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/common.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/dataframe.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/image.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/json_data.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/sparse.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/validate.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/info.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/py.typed +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/read.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/storage/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/util/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/util/names.py +0 -0
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# so the distribution is published as `pyadata-cli`. The import package
|
|
4
4
|
# and the command are both still `adata`.
|
|
5
5
|
name = "pyadata-cli"
|
|
6
|
-
version = "0.7.
|
|
6
|
+
version = "0.7.1"
|
|
7
7
|
description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores"
|
|
8
8
|
readme = "README.md"
|
|
9
9
|
requires-python = ">=3.12"
|
|
@@ -347,6 +347,10 @@ def subset(
|
|
|
347
347
|
adata subset data.h5ad -o out.h5ad -q "n_counts > 1000 and cluster in A,B"
|
|
348
348
|
adata subset data.h5ad -o out.h5ad --var-query "highly_variable == True"
|
|
349
349
|
"""
|
|
350
|
+
if zarr_format is not None and zarr_format not in (2, 3):
|
|
351
|
+
console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.")
|
|
352
|
+
raise typer.Exit(code=1)
|
|
353
|
+
|
|
350
354
|
if obs is None and var is None and obs_query is None and var_query is None:
|
|
351
355
|
console.print(
|
|
352
356
|
"[bold red]Error:[/] Provide at least one of --obs, --var, "
|
|
@@ -433,7 +437,9 @@ def concat(
|
|
|
433
437
|
1024, "--chunk", "-C", help="Row chunk size while streaming"
|
|
434
438
|
),
|
|
435
439
|
zarr_format: Optional[int] = typer.Option(
|
|
436
|
-
None,
|
|
440
|
+
None,
|
|
441
|
+
"--zarr-format",
|
|
442
|
+
help="Zarr spec version to write (defaults to the inputs' when they agree)",
|
|
437
443
|
),
|
|
438
444
|
) -> None:
|
|
439
445
|
"""
|
|
@@ -447,6 +453,10 @@ def concat(
|
|
|
447
453
|
adata concat *.h5ad -o merged.h5ad --join outer --label sample
|
|
448
454
|
adata concat a.h5ad b.h5ad -o m.h5ad --keys a,b --index-unique - --uns-merge same
|
|
449
455
|
"""
|
|
456
|
+
if zarr_format is not None and zarr_format not in (2, 3):
|
|
457
|
+
console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.")
|
|
458
|
+
raise typer.Exit(code=1)
|
|
459
|
+
|
|
450
460
|
for name, value in (("--merge", merge), ("--uns-merge", uns_merge)):
|
|
451
461
|
if value is not None and value not in MERGE_CHOICES:
|
|
452
462
|
console.print(
|
|
@@ -1129,9 +1139,9 @@ def export_sparse(
|
|
|
1129
1139
|
Export a sparse matrix (CSR/CSC) to Matrix Market (.mtx) format.
|
|
1130
1140
|
|
|
1131
1141
|
Examples:
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1142
|
+
adata export sparse data.h5ad X -o matrix.mtx
|
|
1143
|
+
adata export sparse data.h5ad layers/counts -o counts.mtx
|
|
1144
|
+
adata export sparse data.h5ad X --head 100
|
|
1135
1145
|
"""
|
|
1136
1146
|
|
|
1137
1147
|
try:
|
|
@@ -1179,8 +1189,8 @@ def export_dict(
|
|
|
1179
1189
|
Export a dict/group or scalar to JSON format.
|
|
1180
1190
|
|
|
1181
1191
|
Examples:
|
|
1182
|
-
|
|
1183
|
-
|
|
1192
|
+
adata export dict data.h5ad uns metadata.json
|
|
1193
|
+
adata export dict data.h5ad uns/colors colors.json
|
|
1184
1194
|
"""
|
|
1185
1195
|
|
|
1186
1196
|
try:
|
|
@@ -1219,7 +1229,7 @@ def export_image(
|
|
|
1219
1229
|
The array should be 2D (H,W) or 3D (H,W,C) with C in {1,3,4}.
|
|
1220
1230
|
|
|
1221
1231
|
Examples:
|
|
1222
|
-
|
|
1232
|
+
adata export image data.h5ad uns/spatial/image -o tissue.png
|
|
1223
1233
|
"""
|
|
1224
1234
|
|
|
1225
1235
|
try:
|
|
@@ -1363,8 +1373,8 @@ def import_array(
|
|
|
1363
1373
|
Dimensions are validated against existing obs/var.
|
|
1364
1374
|
|
|
1365
1375
|
Examples:
|
|
1366
|
-
|
|
1367
|
-
|
|
1376
|
+
adata import array data.h5ad obsm/X_pca pca.npy -o output.h5ad
|
|
1377
|
+
adata import array data.h5ad X matrix.npy --inplace
|
|
1368
1378
|
"""
|
|
1369
1379
|
from adata.commands.import_data import _import_npy
|
|
1370
1380
|
|
|
@@ -1419,8 +1429,8 @@ def import_sparse(
|
|
|
1419
1429
|
Dimensions are validated against existing obs/var.
|
|
1420
1430
|
|
|
1421
1431
|
Examples:
|
|
1422
|
-
|
|
1423
|
-
|
|
1432
|
+
adata import sparse data.h5ad X matrix.mtx -o output.h5ad
|
|
1433
|
+
adata import sparse data.h5ad layers/counts counts.mtx --inplace
|
|
1424
1434
|
"""
|
|
1425
1435
|
from adata.commands.import_data import _import_mtx
|
|
1426
1436
|
|
|
@@ -1473,8 +1483,8 @@ def import_dict(
|
|
|
1473
1483
|
Import a JSON file into uns or other dict-like groups.
|
|
1474
1484
|
|
|
1475
1485
|
Examples:
|
|
1476
|
-
|
|
1477
|
-
|
|
1486
|
+
adata import dict data.h5ad uns/metadata config.json -o output.h5ad
|
|
1487
|
+
adata import dict data.h5ad uns settings.json --inplace
|
|
1478
1488
|
"""
|
|
1479
1489
|
from adata.commands.import_data import _import_json
|
|
1480
1490
|
|
|
@@ -20,16 +20,10 @@ from adata.conversion import (
|
|
|
20
20
|
detect_format,
|
|
21
21
|
output_format,
|
|
22
22
|
)
|
|
23
|
+
from adata.util.path import same_path
|
|
23
24
|
from adata.util.staging import staged_outputs
|
|
24
25
|
|
|
25
26
|
|
|
26
|
-
def _same_path(left: Path, right: Path) -> bool:
|
|
27
|
-
try:
|
|
28
|
-
return left.resolve() == right.resolve()
|
|
29
|
-
except OSError:
|
|
30
|
-
return False
|
|
31
|
-
|
|
32
|
-
|
|
33
27
|
def convert_store(
|
|
34
28
|
file: Path,
|
|
35
29
|
output: Path,
|
|
@@ -56,7 +50,7 @@ def convert_store(
|
|
|
56
50
|
|
|
57
51
|
plan = converter.plan(file, output, src_format, dst_format, options)
|
|
58
52
|
for out in plan.outputs:
|
|
59
|
-
if
|
|
53
|
+
if same_path(file, out):
|
|
60
54
|
# Writing a Zarr store over its own source empties it before a
|
|
61
55
|
# single element has been read.
|
|
62
56
|
raise ValueError(f"Output path is the input: {out}")
|
|
@@ -2,13 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
Writes a whole new store rather than editing one in place, for the same
|
|
4
4
|
reason `subset` does: a conversion that fails half way leaves the original
|
|
5
|
-
untouched
|
|
6
|
-
|
|
5
|
+
untouched. The new store is written beside its destination and renamed into
|
|
6
|
+
place only once complete (see `adata.util.staging`), so `--inplace` replaces a
|
|
7
|
+
finished file rather than editing a live one, and the original is deleted only
|
|
8
|
+
after its successor is in place.
|
|
7
9
|
"""
|
|
8
10
|
|
|
9
11
|
from __future__ import annotations
|
|
10
12
|
|
|
11
|
-
import shutil
|
|
12
13
|
from pathlib import Path
|
|
13
14
|
from typing import Any, List, Optional
|
|
14
15
|
|
|
@@ -25,15 +26,16 @@ from adata.core.recast import (
|
|
|
25
26
|
from adata.elements import spec
|
|
26
27
|
from adata.elements.write import ensure_anndata_skeleton
|
|
27
28
|
from adata.storage import copy_tree, detect_backend, is_group, open_store
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
#: every 2-D array in the file: obsm/varm hold embeddings, whose dtype is
|
|
31
|
-
#: rarely what anyone is trying to shrink. Those still convert by path.
|
|
32
|
-
ALL_PREFIXES = ("X", "layers", "raw/X")
|
|
33
|
-
|
|
29
|
+
from adata.util.path import same_path
|
|
30
|
+
from adata.util.staging import staged_outputs
|
|
34
31
|
|
|
35
32
|
def discover_matrices(root: Any) -> List[str]:
|
|
36
|
-
"""Entry paths `--all` resolves to, in a stable order.
|
|
33
|
+
"""Entry paths `--all` resolves to, in a stable order.
|
|
34
|
+
|
|
35
|
+
The matrices an AnnData object is built from, not every 2-D array in the
|
|
36
|
+
file: obsm/varm hold embeddings, whose dtype is rarely what anyone is
|
|
37
|
+
trying to shrink. Those still convert by path.
|
|
38
|
+
"""
|
|
37
39
|
found: List[str] = []
|
|
38
40
|
if "X" in root:
|
|
39
41
|
found.append("X")
|
|
@@ -44,14 +46,6 @@ def discover_matrices(root: Any) -> List[str]:
|
|
|
44
46
|
return found
|
|
45
47
|
|
|
46
48
|
|
|
47
|
-
def _same_path(left: Path, right: Path) -> bool:
|
|
48
|
-
"""Do these name the same store, through symlinks and `..` alike?"""
|
|
49
|
-
try:
|
|
50
|
-
return left.resolve() == right.resolve()
|
|
51
|
-
except OSError: # pragma: no cover - unresolvable path
|
|
52
|
-
return left.absolute() == right.absolute()
|
|
53
|
-
|
|
54
|
-
|
|
55
49
|
def _resolve(root: Any, path: str) -> Any:
|
|
56
50
|
obj = root
|
|
57
51
|
for part in path.split("/"):
|
|
@@ -96,7 +90,7 @@ def recast_store(
|
|
|
96
90
|
else None
|
|
97
91
|
)
|
|
98
92
|
|
|
99
|
-
if output is not None and not inplace and
|
|
93
|
+
if output is not None and not inplace and same_path(file, output):
|
|
100
94
|
# Opening the destination "w" clears it while the source is still
|
|
101
95
|
# being read from it. HDF5 refuses; Zarr does not, and quietly
|
|
102
96
|
# produced an empty store where the data used to be.
|
|
@@ -105,69 +99,54 @@ def recast_store(
|
|
|
105
99
|
"it, which writes to a temporary file first."
|
|
106
100
|
)
|
|
107
101
|
|
|
108
|
-
if inplace
|
|
109
|
-
backend = detect_backend(file)
|
|
110
|
-
if backend == "zarr":
|
|
111
|
-
base = file.stem if file.suffix else file.name
|
|
112
|
-
dst_path = file.with_name(f"{base}.recast-tmp.zarr")
|
|
113
|
-
else:
|
|
114
|
-
dst_path = file.with_name(f"{file.name}.recast-tmp")
|
|
115
|
-
if dst_path.exists():
|
|
116
|
-
raise FileExistsError(f"Temporary path already exists: {dst_path}")
|
|
117
|
-
else:
|
|
118
|
-
dst_path = output
|
|
102
|
+
dst_path = file if inplace else Path(output)
|
|
119
103
|
|
|
120
104
|
if zarr_format is None and detect_backend(file) == "zarr":
|
|
121
105
|
with open_store(file, "r") as probe:
|
|
122
106
|
zarr_format = probe.zarr_format
|
|
123
107
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
108
|
+
# The source is opened inside the staging block, so it is closed before
|
|
109
|
+
# `--inplace` renames it aside.
|
|
110
|
+
with staged_outputs([dst_path]) as (temp,):
|
|
111
|
+
with open_store(file, "r") as src_store:
|
|
112
|
+
src = src_store.root
|
|
113
|
+
targets = discover_matrices(src) if convert_all else list(entries or [])
|
|
114
|
+
if not targets:
|
|
115
|
+
raise ValueError(
|
|
116
|
+
"No matrices selected. Name one (e.g. `X`) or pass --all."
|
|
117
|
+
)
|
|
118
|
+
# Every check first, before the destination exists. A refusal must
|
|
119
|
+
# not leave behind a store holding a copy of obs and var and nothing
|
|
120
|
+
# else -- the caller cannot tell that from a finished conversion.
|
|
121
|
+
plans = {
|
|
122
|
+
path: plan_conversion(
|
|
123
|
+
_resolve(src, path),
|
|
124
|
+
path,
|
|
125
|
+
dtype=data_dtype,
|
|
126
|
+
index_dtype=index_dtype,
|
|
127
|
+
layout=layout,
|
|
128
|
+
chunk=chunk,
|
|
129
|
+
force=force,
|
|
130
|
+
console=console,
|
|
131
|
+
)
|
|
132
|
+
for path in targets
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
console.print(
|
|
136
|
+
f"[cyan]Converting {len(targets)} matrix/matrices:[/] "
|
|
137
|
+
+ ", ".join(targets)
|
|
144
138
|
)
|
|
145
|
-
for path in targets
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
console.print(
|
|
149
|
-
f"[cyan]Converting {len(targets)} matrix/matrices:[/] "
|
|
150
|
-
+ ", ".join(targets)
|
|
151
|
-
)
|
|
152
139
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
140
|
+
with open_store(temp, "w", zarr_format=zarr_format) as dst_store:
|
|
141
|
+
dst = dst_store.root
|
|
142
|
+
_write(
|
|
143
|
+
src, dst, plans,
|
|
144
|
+
chunk=chunk, chunk_rows=chunk_rows, in_memory=in_memory,
|
|
145
|
+
console=console,
|
|
146
|
+
)
|
|
147
|
+
ensure_anndata_skeleton(dst)
|
|
161
148
|
|
|
162
149
|
if inplace:
|
|
163
|
-
if file.is_dir():
|
|
164
|
-
shutil.rmtree(file)
|
|
165
|
-
elif file.exists():
|
|
166
|
-
file.unlink()
|
|
167
|
-
if dst_path.is_dir():
|
|
168
|
-
shutil.move(str(dst_path), str(file))
|
|
169
|
-
else:
|
|
170
|
-
dst_path.replace(file)
|
|
171
150
|
console.print(f"[green]Converted[/] {file}")
|
|
172
151
|
else:
|
|
173
152
|
console.print(f"[green]Wrote[/] {dst_path}")
|
|
@@ -4,6 +4,11 @@ Requested in issue #2, modelled on cellgeni/scraft's `split_h5ad` but
|
|
|
4
4
|
streaming: the source is never loaded into memory, only the column being split
|
|
5
5
|
on is read, and each output is produced by the same subset machinery used by
|
|
6
6
|
`adata subset`, fanned out so every matrix is read once for all groups.
|
|
7
|
+
|
|
8
|
+
Every output is written beside its destination and renamed into place only
|
|
9
|
+
once all of them are complete, so a split that fails leaves no partial stores
|
|
10
|
+
(and existing outputs as they were). The manifest is written last, so it
|
|
11
|
+
exists only when everything it lists does.
|
|
7
12
|
"""
|
|
8
13
|
|
|
9
14
|
from __future__ import annotations
|
|
@@ -19,6 +24,8 @@ from adata.core.select import group_indices
|
|
|
19
24
|
from adata.core.subset import split_h5ad
|
|
20
25
|
from adata.storage import detect_backend, open_store
|
|
21
26
|
from adata.util.names import unique_names
|
|
27
|
+
from adata.util.path import same_path
|
|
28
|
+
from adata.util.staging import staged_outputs
|
|
22
29
|
|
|
23
30
|
|
|
24
31
|
def split_store(
|
|
@@ -78,6 +85,13 @@ def split_store(
|
|
|
78
85
|
for label, out_path, count in planned:
|
|
79
86
|
console.print(f" {label!r} -> {out_path} ({count} {axis})")
|
|
80
87
|
|
|
88
|
+
for _, out_path, _ in planned:
|
|
89
|
+
if same_path(file, out_path):
|
|
90
|
+
raise ValueError(
|
|
91
|
+
f"Output path is the input: {out_path}. Choose another "
|
|
92
|
+
"--output-dir."
|
|
93
|
+
)
|
|
94
|
+
|
|
81
95
|
if dry_run:
|
|
82
96
|
console.print("[yellow]Dry run: nothing written.[/]")
|
|
83
97
|
return planned
|
|
@@ -86,20 +100,21 @@ def split_store(
|
|
|
86
100
|
|
|
87
101
|
# One call for every group, so each matrix is read once rather than once
|
|
88
102
|
# per group.
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
+
with staged_outputs([out_path for _, out_path, _ in planned]) as temps:
|
|
104
|
+
split_h5ad(
|
|
105
|
+
file,
|
|
106
|
+
[
|
|
107
|
+
(
|
|
108
|
+
temp,
|
|
109
|
+
np.sort(groups[label]) if axis == "obs" else None,
|
|
110
|
+
np.sort(groups[label]) if axis == "var" else None,
|
|
111
|
+
)
|
|
112
|
+
for (label, _, _), temp in zip(planned, temps)
|
|
113
|
+
],
|
|
114
|
+
chunk_rows=chunk_rows,
|
|
115
|
+
console=console,
|
|
116
|
+
zarr_format=zarr_format,
|
|
117
|
+
)
|
|
103
118
|
|
|
104
119
|
if manifest:
|
|
105
120
|
_write_manifest(file, output_dir, column, planned, console)
|
|
@@ -43,9 +43,11 @@ from adata.storage import (
|
|
|
43
43
|
copy_tree,
|
|
44
44
|
create_dataset,
|
|
45
45
|
is_dataset,
|
|
46
|
+
Store,
|
|
46
47
|
is_group,
|
|
47
48
|
open_store,
|
|
48
49
|
)
|
|
50
|
+
from adata.util.staging import staged_outputs
|
|
49
51
|
|
|
50
52
|
MERGE_STRATEGIES = ("same", "unique", "first", "only")
|
|
51
53
|
#: What the CLI accepts. "drop" is the default and maps to no merge at all.
|
|
@@ -827,6 +829,17 @@ def _hashable(value: Any) -> Any:
|
|
|
827
829
|
return value
|
|
828
830
|
|
|
829
831
|
|
|
832
|
+
def _inputs_zarr_format(stores: Sequence[Store]) -> Optional[int]:
|
|
833
|
+
"""The Zarr version the inputs agree on, or None to take the default.
|
|
834
|
+
|
|
835
|
+
A Zarr output follows its inputs, as every other command's follows its
|
|
836
|
+
source, so v2 inputs are not silently upgraded. h5ad inputs have no
|
|
837
|
+
version and do not count; Zarr inputs of both versions get the default.
|
|
838
|
+
"""
|
|
839
|
+
versions = {s.zarr_format for s in stores} - {None}
|
|
840
|
+
return versions.pop() if len(versions) == 1 else None
|
|
841
|
+
|
|
842
|
+
|
|
830
843
|
def concat_on_disk(
|
|
831
844
|
files: Sequence[Path],
|
|
832
845
|
output: Path,
|
|
@@ -906,7 +919,14 @@ def concat_on_disk(
|
|
|
906
919
|
# half-written store behind.
|
|
907
920
|
check_matrix_encodings(roots, console)
|
|
908
921
|
|
|
909
|
-
|
|
922
|
+
if zarr_format is None:
|
|
923
|
+
zarr_format = _inputs_zarr_format(stores)
|
|
924
|
+
|
|
925
|
+
# Written beside the output and renamed into place once complete, so
|
|
926
|
+
# a failure part way leaves nothing that reads as a finished store.
|
|
927
|
+
with staged_outputs([output]) as (temp,), open_store(
|
|
928
|
+
temp, "w", zarr_format=zarr_format
|
|
929
|
+
) as dst_store:
|
|
910
930
|
dst = dst_store.root
|
|
911
931
|
_write_obs(
|
|
912
932
|
dst, roots, target_obs, obs_counts, join, label, input_keys, console
|
|
@@ -5,7 +5,6 @@ from __future__ import annotations
|
|
|
5
5
|
from contextlib import ExitStack
|
|
6
6
|
from dataclasses import dataclass
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
import shutil
|
|
9
8
|
from typing import Optional, Set, Tuple, List, Dict, Any
|
|
10
9
|
|
|
11
10
|
import numpy as np
|
|
@@ -43,6 +42,8 @@ from adata.storage import (
|
|
|
43
42
|
with_compression,
|
|
44
43
|
zarr_format_of,
|
|
45
44
|
)
|
|
45
|
+
from adata.util.path import same_path
|
|
46
|
+
from adata.util.staging import staged_outputs
|
|
46
47
|
|
|
47
48
|
|
|
48
49
|
def _target_backend(dst_group: Any) -> str:
|
|
@@ -952,18 +953,15 @@ def subset_h5ad(
|
|
|
952
953
|
if not inplace and output is None:
|
|
953
954
|
raise ValueError("Output file is required unless --inplace is specified.")
|
|
954
955
|
|
|
955
|
-
if inplace:
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
dst_path = tmp_path
|
|
965
|
-
else:
|
|
966
|
-
dst_path = output
|
|
956
|
+
if not inplace and same_path(file, output):
|
|
957
|
+
# Opening the output "w" would empty a Zarr source before a single
|
|
958
|
+
# element had been read from it.
|
|
959
|
+
raise ValueError(
|
|
960
|
+
f"Output path is the input: {output}. Use --inplace to replace "
|
|
961
|
+
"it, which writes to a temporary file first."
|
|
962
|
+
)
|
|
963
|
+
|
|
964
|
+
dst_path = file if inplace else Path(output)
|
|
967
965
|
|
|
968
966
|
if zarr_format is None and detect_backend(file) == "zarr":
|
|
969
967
|
with open_store(file, "r") as probe:
|
|
@@ -975,12 +973,17 @@ def subset_h5ad(
|
|
|
975
973
|
status = console.status("[magenta]Opening files...[/]")
|
|
976
974
|
status.start()
|
|
977
975
|
try:
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
) as
|
|
976
|
+
# The output is written beside its destination and renamed into place
|
|
977
|
+
# only once complete, so a failure leaves any existing output (or,
|
|
978
|
+
# with --inplace, the source) as it was. The source is opened inside
|
|
979
|
+
# the staging block, so it is closed before it is renamed aside.
|
|
980
|
+
with staged_outputs([dst_path]) as (temp,), open_store(
|
|
981
|
+
file, "r"
|
|
982
|
+
) as src_store:
|
|
981
983
|
src = src_store.root
|
|
982
|
-
dst = dst_store.root
|
|
983
984
|
|
|
985
|
+
# The selection is resolved before the output exists: a query
|
|
986
|
+
# that fails must not leave an empty store behind.
|
|
984
987
|
obs_idx = (
|
|
985
988
|
obs_indices
|
|
986
989
|
if obs_indices is not None
|
|
@@ -1000,26 +1003,16 @@ def subset_h5ad(
|
|
|
1000
1003
|
|
|
1001
1004
|
status.stop()
|
|
1002
1005
|
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1006
|
+
with open_store(temp, "w", zarr_format=zarr_format) as dst_store:
|
|
1007
|
+
_write_targets(
|
|
1008
|
+
src,
|
|
1009
|
+
[Target(dst_store.root, obs_idx, var_idx, var_keep)],
|
|
1010
|
+
chunk_rows=chunk_rows,
|
|
1011
|
+
console=console,
|
|
1012
|
+
)
|
|
1009
1013
|
finally:
|
|
1010
1014
|
status.stop()
|
|
1011
1015
|
|
|
1012
|
-
if inplace:
|
|
1013
|
-
if file.exists():
|
|
1014
|
-
if file.is_dir():
|
|
1015
|
-
shutil.rmtree(file)
|
|
1016
|
-
else:
|
|
1017
|
-
file.unlink()
|
|
1018
|
-
if dst_path.is_dir():
|
|
1019
|
-
shutil.move(str(dst_path), str(file))
|
|
1020
|
-
else:
|
|
1021
|
-
dst_path.replace(file)
|
|
1022
|
-
|
|
1023
1016
|
|
|
1024
1017
|
def _open_output_limit() -> int:
|
|
1025
1018
|
"""How many output stores `split_h5ad` may hold open at once.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def norm_path(path: str) -> str:
|
|
7
|
+
"""Normalize object paths used inside h5ad/zarr stores."""
|
|
8
|
+
value = path.strip()
|
|
9
|
+
if not value:
|
|
10
|
+
raise ValueError("Object path must be non-empty.")
|
|
11
|
+
return value.lstrip("/")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def same_path(left: Path, right: Path) -> bool:
|
|
15
|
+
"""Do these name the same store, through symlinks and `..` alike?"""
|
|
16
|
+
try:
|
|
17
|
+
return Path(left).resolve() == Path(right).resolve()
|
|
18
|
+
except OSError: # pragma: no cover - unresolvable path
|
|
19
|
+
return Path(left).absolute() == Path(right).absolute()
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Write outputs beside their destination, and move them into place at the end.
|
|
2
|
+
|
|
3
|
+
A command that fails half way must not leave a half-written store where the
|
|
4
|
+
output should be: the next command to open it would read whatever got written
|
|
5
|
+
as if it were the whole thing. Writing to a temporary sibling and renaming it
|
|
6
|
+
only on success means the output either appears whole or not at all.
|
|
7
|
+
|
|
8
|
+
An existing output is renamed aside (`out.adata-old.zarr`) before its
|
|
9
|
+
successor is renamed in, and deleted only once every output is in place. If a
|
|
10
|
+
rename fails part way, the outputs already swapped are put back. A run killed
|
|
11
|
+
between the two renames leaves the old data under the `adata-old` name, never
|
|
12
|
+
nowhere, and the next run refuses to start until that has been dealt with.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import shutil
|
|
18
|
+
from contextlib import contextmanager
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Iterator, List, Sequence, Tuple
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _sibling(path: Path, tag: str) -> Path:
|
|
24
|
+
"""`out.zarr` -> `out.<tag>.zarr`, keeping the suffix that picks the backend."""
|
|
25
|
+
return path.with_name(f"{path.stem}.{tag}{path.suffix}")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _temp_for(path: Path) -> Path:
|
|
29
|
+
temp = _sibling(path, "adata-tmp")
|
|
30
|
+
if path.is_dir() and path.suffix != ".zarr":
|
|
31
|
+
# A Zarr store named without the suffix. A temporary named the same
|
|
32
|
+
# way would not exist yet, and so be opened as HDF5.
|
|
33
|
+
temp = temp.with_name(f"{temp.name}.zarr")
|
|
34
|
+
return temp
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _remove(path: Path) -> None:
|
|
38
|
+
if path.is_dir() and not path.is_symlink():
|
|
39
|
+
shutil.rmtree(path)
|
|
40
|
+
elif path.exists() or path.is_symlink():
|
|
41
|
+
path.unlink()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _exists(path: Path) -> bool:
|
|
45
|
+
return path.exists() or path.is_symlink()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@contextmanager
|
|
49
|
+
def staged_outputs(outputs: Sequence[Path]) -> Iterator[List[Path]]:
|
|
50
|
+
"""Yield temporary paths to write `outputs` to; move them into place after.
|
|
51
|
+
|
|
52
|
+
On success each output is swapped in: an existing one is set aside first
|
|
53
|
+
and deleted only once every new output is in place. On failure the
|
|
54
|
+
temporary paths are deleted and the outputs are left as they were.
|
|
55
|
+
"""
|
|
56
|
+
outputs = [Path(p) for p in outputs]
|
|
57
|
+
temps = [_temp_for(p) for p in outputs]
|
|
58
|
+
backups = [_sibling(p, "adata-old") for p in outputs]
|
|
59
|
+
for leftover in temps + backups:
|
|
60
|
+
if _exists(leftover):
|
|
61
|
+
raise FileExistsError(
|
|
62
|
+
f"'{leftover}' already exists, probably left by an interrupted "
|
|
63
|
+
"run. Check what it holds, remove it, and try again."
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
try:
|
|
67
|
+
yield temps
|
|
68
|
+
except BaseException:
|
|
69
|
+
for temp in temps:
|
|
70
|
+
_remove(temp)
|
|
71
|
+
raise
|
|
72
|
+
|
|
73
|
+
swapped: List[Tuple[Path, Path, bool]] = []
|
|
74
|
+
try:
|
|
75
|
+
for output, temp, backup in zip(outputs, temps, backups):
|
|
76
|
+
had_output = _exists(output)
|
|
77
|
+
if had_output:
|
|
78
|
+
output.rename(backup)
|
|
79
|
+
try:
|
|
80
|
+
temp.rename(output)
|
|
81
|
+
except BaseException:
|
|
82
|
+
if had_output:
|
|
83
|
+
backup.rename(output)
|
|
84
|
+
raise
|
|
85
|
+
swapped.append((output, backup, had_output))
|
|
86
|
+
except BaseException:
|
|
87
|
+
# Put back what was already swapped, newest first, and drop the
|
|
88
|
+
# temporaries that never made it.
|
|
89
|
+
for output, backup, had_output in reversed(swapped):
|
|
90
|
+
_remove(output)
|
|
91
|
+
if had_output:
|
|
92
|
+
backup.rename(output)
|
|
93
|
+
for temp in temps:
|
|
94
|
+
_remove(temp)
|
|
95
|
+
raise
|
|
96
|
+
|
|
97
|
+
for _, backup, had_output in swapped:
|
|
98
|
+
if had_output:
|
|
99
|
+
_remove(backup)
|
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
"""Write outputs beside their destination, and move them into place at the end.
|
|
2
|
-
|
|
3
|
-
A conversion that fails half way must not leave a half-written store where
|
|
4
|
-
the output should be: the next command to open it would read whatever got
|
|
5
|
-
written as if it were the whole thing. Writing to a temporary sibling and
|
|
6
|
-
renaming it only on success means the output either appears whole or not at
|
|
7
|
-
all, and an existing output is replaced only once its successor is complete.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
|
-
from __future__ import annotations
|
|
11
|
-
|
|
12
|
-
import shutil
|
|
13
|
-
from contextlib import contextmanager
|
|
14
|
-
from pathlib import Path
|
|
15
|
-
from typing import Iterator, List, Sequence
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def _sibling(path: Path, tag: str) -> Path:
|
|
19
|
-
"""`out.zarr` -> `out.<tag>.zarr`, keeping the suffix that picks the backend."""
|
|
20
|
-
return path.with_name(f"{path.stem}.{tag}{path.suffix}")
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def _remove(path: Path) -> None:
|
|
24
|
-
if path.is_dir() and not path.is_symlink():
|
|
25
|
-
shutil.rmtree(path)
|
|
26
|
-
elif path.exists() or path.is_symlink():
|
|
27
|
-
path.unlink()
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
@contextmanager
|
|
31
|
-
def staged_outputs(outputs: Sequence[Path]) -> Iterator[List[Path]]:
|
|
32
|
-
"""Yield temporary paths to write `outputs` to; move them into place after.
|
|
33
|
-
|
|
34
|
-
On success each output is swapped in: an existing one is set aside first
|
|
35
|
-
and deleted only once every new output is in place. On failure the
|
|
36
|
-
temporary paths are deleted and the outputs are left as they were.
|
|
37
|
-
"""
|
|
38
|
-
outputs = [Path(p) for p in outputs]
|
|
39
|
-
temps = [_sibling(p, "adata-tmp") for p in outputs]
|
|
40
|
-
for temp in temps:
|
|
41
|
-
if temp.exists():
|
|
42
|
-
raise FileExistsError(
|
|
43
|
-
f"'{temp}' already exists, probably left by an interrupted run. "
|
|
44
|
-
"Remove it and try again."
|
|
45
|
-
)
|
|
46
|
-
|
|
47
|
-
try:
|
|
48
|
-
yield temps
|
|
49
|
-
except BaseException:
|
|
50
|
-
for temp in temps:
|
|
51
|
-
_remove(temp)
|
|
52
|
-
raise
|
|
53
|
-
|
|
54
|
-
backups = []
|
|
55
|
-
for output, temp in zip(outputs, temps):
|
|
56
|
-
if output.exists():
|
|
57
|
-
backup = _sibling(output, "adata-old")
|
|
58
|
-
_remove(backup)
|
|
59
|
-
output.rename(backup)
|
|
60
|
-
backups.append(backup)
|
|
61
|
-
temp.rename(output)
|
|
62
|
-
for backup in backups:
|
|
63
|
-
_remove(backup)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|