pyadata-cli 0.7.0__tar.gz → 0.7.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/PKG-INFO +1 -1
  2. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/pyproject.toml +1 -1
  3. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/cli.py +23 -13
  4. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/convert.py +2 -8
  5. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/recast.py +52 -73
  6. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/split.py +29 -14
  7. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/concat.py +21 -1
  8. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/subset.py +27 -34
  9. pyadata_cli-0.7.1/src/adata/util/path.py +19 -0
  10. pyadata_cli-0.7.1/src/adata/util/staging.py +99 -0
  11. pyadata_cli-0.7.0/src/adata/util/path.py +0 -9
  12. pyadata_cli-0.7.0/src/adata/util/staging.py +0 -63
  13. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/LICENSE +0 -0
  14. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/README.md +0 -0
  15. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/__init__.py +0 -0
  16. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/__init__.py +0 -0
  17. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/concat.py +0 -0
  18. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/create.py +0 -0
  19. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/export.py +0 -0
  20. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/import_data.py +0 -0
  21. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/info.py +0 -0
  22. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/ls.py +0 -0
  23. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/commands/subset.py +0 -0
  24. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/conversion/__init__.py +0 -0
  25. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/conversion/anndata_copy.py +0 -0
  26. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/conversion/base.py +0 -0
  27. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/conversion/tenx.py +0 -0
  28. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/__init__.py +0 -0
  29. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/info.py +0 -0
  30. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/query.py +0 -0
  31. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/read.py +0 -0
  32. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/recast.py +0 -0
  33. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/core/select.py +0 -0
  34. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/elements/__init__.py +0 -0
  35. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/elements/read.py +0 -0
  36. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/elements/spec.py +0 -0
  37. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/elements/strings.py +0 -0
  38. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/elements/write.py +0 -0
  39. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/__init__.py +0 -0
  40. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/array.py +0 -0
  41. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/common.py +0 -0
  42. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/dataframe.py +0 -0
  43. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/image.py +0 -0
  44. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/json_data.py +0 -0
  45. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/sparse.py +0 -0
  46. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/formats/validate.py +0 -0
  47. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/info.py +0 -0
  48. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/py.typed +0 -0
  49. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/read.py +0 -0
  50. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/storage/__init__.py +0 -0
  51. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/util/__init__.py +0 -0
  52. {pyadata_cli-0.7.0 → pyadata_cli-0.7.1}/src/adata/util/names.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pyadata-cli
3
- Version: 0.7.0
3
+ Version: 0.7.1
4
4
  Summary: Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores
5
5
  Keywords: anndata,h5ad,zarr,bioinformatics,single-cell,cli
6
6
  Author: Aljes Binkevich
@@ -3,7 +3,7 @@
3
3
  # so the distribution is published as `pyadata-cli`. The import package
4
4
  # and the command are both still `adata`.
5
5
  name = "pyadata-cli"
6
- version = "0.7.0"
6
+ version = "0.7.1"
7
7
  description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores"
8
8
  readme = "README.md"
9
9
  requires-python = ">=3.12"
@@ -347,6 +347,10 @@ def subset(
347
347
  adata subset data.h5ad -o out.h5ad -q "n_counts > 1000 and cluster in A,B"
348
348
  adata subset data.h5ad -o out.h5ad --var-query "highly_variable == True"
349
349
  """
350
+ if zarr_format is not None and zarr_format not in (2, 3):
351
+ console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.")
352
+ raise typer.Exit(code=1)
353
+
350
354
  if obs is None and var is None and obs_query is None and var_query is None:
351
355
  console.print(
352
356
  "[bold red]Error:[/] Provide at least one of --obs, --var, "
@@ -433,7 +437,9 @@ def concat(
433
437
  1024, "--chunk", "-C", help="Row chunk size while streaming"
434
438
  ),
435
439
  zarr_format: Optional[int] = typer.Option(
436
- None, "--zarr-format", help="Zarr spec version to write (2 or 3)"
440
+ None,
441
+ "--zarr-format",
442
+ help="Zarr spec version to write (defaults to the inputs' when they agree)",
437
443
  ),
438
444
  ) -> None:
439
445
  """
@@ -447,6 +453,10 @@ def concat(
447
453
  adata concat *.h5ad -o merged.h5ad --join outer --label sample
448
454
  adata concat a.h5ad b.h5ad -o m.h5ad --keys a,b --index-unique - --uns-merge same
449
455
  """
456
+ if zarr_format is not None and zarr_format not in (2, 3):
457
+ console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.")
458
+ raise typer.Exit(code=1)
459
+
450
460
  for name, value in (("--merge", merge), ("--uns-merge", uns_merge)):
451
461
  if value is not None and value not in MERGE_CHOICES:
452
462
  console.print(
@@ -1129,9 +1139,9 @@ def export_sparse(
1129
1139
  Export a sparse matrix (CSR/CSC) to Matrix Market (.mtx) format.
1130
1140
 
1131
1141
  Examples:
1132
- h5ad export sparse data.h5ad X matrix.mtx
1133
- h5ad export sparse data.h5ad layers/counts counts.mtx
1134
- h5ad export sparse data.h5ad X --head 100
1142
+ adata export sparse data.h5ad X -o matrix.mtx
1143
+ adata export sparse data.h5ad layers/counts -o counts.mtx
1144
+ adata export sparse data.h5ad X --head 100
1135
1145
  """
1136
1146
 
1137
1147
  try:
@@ -1179,8 +1189,8 @@ def export_dict(
1179
1189
  Export a dict/group or scalar to JSON format.
1180
1190
 
1181
1191
  Examples:
1182
- h5ad export dict data.h5ad uns metadata.json
1183
- h5ad export dict data.h5ad uns/colors colors.json
1192
+ adata export dict data.h5ad uns metadata.json
1193
+ adata export dict data.h5ad uns/colors colors.json
1184
1194
  """
1185
1195
 
1186
1196
  try:
@@ -1219,7 +1229,7 @@ def export_image(
1219
1229
  The array should be 2D (H,W) or 3D (H,W,C) with C in {1,3,4}.
1220
1230
 
1221
1231
  Examples:
1222
- h5ad export image data.h5ad uns/spatial/image tissue.png
1232
+ adata export image data.h5ad uns/spatial/image -o tissue.png
1223
1233
  """
1224
1234
 
1225
1235
  try:
@@ -1363,8 +1373,8 @@ def import_array(
1363
1373
  Dimensions are validated against existing obs/var.
1364
1374
 
1365
1375
  Examples:
1366
- h5ad import array data.h5ad obsm/X_pca pca.npy -o output.h5ad
1367
- h5ad import array data.h5ad X matrix.npy --inplace
1376
+ adata import array data.h5ad obsm/X_pca pca.npy -o output.h5ad
1377
+ adata import array data.h5ad X matrix.npy --inplace
1368
1378
  """
1369
1379
  from adata.commands.import_data import _import_npy
1370
1380
 
@@ -1419,8 +1429,8 @@ def import_sparse(
1419
1429
  Dimensions are validated against existing obs/var.
1420
1430
 
1421
1431
  Examples:
1422
- h5ad import sparse data.h5ad X matrix.mtx -o output.h5ad
1423
- h5ad import sparse data.h5ad layers/counts counts.mtx --inplace
1432
+ adata import sparse data.h5ad X matrix.mtx -o output.h5ad
1433
+ adata import sparse data.h5ad layers/counts counts.mtx --inplace
1424
1434
  """
1425
1435
  from adata.commands.import_data import _import_mtx
1426
1436
 
@@ -1473,8 +1483,8 @@ def import_dict(
1473
1483
  Import a JSON file into uns or other dict-like groups.
1474
1484
 
1475
1485
  Examples:
1476
- h5ad import dict data.h5ad uns/metadata config.json -o output.h5ad
1477
- h5ad import dict data.h5ad uns settings.json --inplace
1486
+ adata import dict data.h5ad uns/metadata config.json -o output.h5ad
1487
+ adata import dict data.h5ad uns settings.json --inplace
1478
1488
  """
1479
1489
  from adata.commands.import_data import _import_json
1480
1490
 
@@ -20,16 +20,10 @@ from adata.conversion import (
20
20
  detect_format,
21
21
  output_format,
22
22
  )
23
+ from adata.util.path import same_path
23
24
  from adata.util.staging import staged_outputs
24
25
 
25
26
 
26
- def _same_path(left: Path, right: Path) -> bool:
27
- try:
28
- return left.resolve() == right.resolve()
29
- except OSError:
30
- return False
31
-
32
-
33
27
  def convert_store(
34
28
  file: Path,
35
29
  output: Path,
@@ -56,7 +50,7 @@ def convert_store(
56
50
 
57
51
  plan = converter.plan(file, output, src_format, dst_format, options)
58
52
  for out in plan.outputs:
59
- if _same_path(file, out):
53
+ if same_path(file, out):
60
54
  # Writing a Zarr store over its own source empties it before a
61
55
  # single element has been read.
62
56
  raise ValueError(f"Output path is the input: {out}")
@@ -2,13 +2,14 @@
2
2
 
3
3
  Writes a whole new store rather than editing one in place, for the same
4
4
  reason `subset` does: a conversion that fails half way leaves the original
5
- untouched, and `--inplace` becomes an atomic swap of a finished file rather
6
- than a partial edit of a live one.
5
+ untouched. The new store is written beside its destination and renamed into
6
+ place only once complete (see `adata.util.staging`), so `--inplace` replaces a
7
+ finished file rather than editing a live one, and the original is deleted only
8
+ after its successor is in place.
7
9
  """
8
10
 
9
11
  from __future__ import annotations
10
12
 
11
- import shutil
12
13
  from pathlib import Path
13
14
  from typing import Any, List, Optional
14
15
 
@@ -25,15 +26,16 @@ from adata.core.recast import (
25
26
  from adata.elements import spec
26
27
  from adata.elements.write import ensure_anndata_skeleton
27
28
  from adata.storage import copy_tree, detect_backend, is_group, open_store
28
-
29
- #: What `--all` means. The matrices an AnnData object is built from, not
30
- #: every 2-D array in the file: obsm/varm hold embeddings, whose dtype is
31
- #: rarely what anyone is trying to shrink. Those still convert by path.
32
- ALL_PREFIXES = ("X", "layers", "raw/X")
33
-
29
+ from adata.util.path import same_path
30
+ from adata.util.staging import staged_outputs
34
31
 
35
32
  def discover_matrices(root: Any) -> List[str]:
36
- """Entry paths `--all` resolves to, in a stable order."""
33
+ """Entry paths `--all` resolves to, in a stable order.
34
+
35
+ The matrices an AnnData object is built from, not every 2-D array in the
36
+ file: obsm/varm hold embeddings, whose dtype is rarely what anyone is
37
+ trying to shrink. Those still convert by path.
38
+ """
37
39
  found: List[str] = []
38
40
  if "X" in root:
39
41
  found.append("X")
@@ -44,14 +46,6 @@ def discover_matrices(root: Any) -> List[str]:
44
46
  return found
45
47
 
46
48
 
47
- def _same_path(left: Path, right: Path) -> bool:
48
- """Do these name the same store, through symlinks and `..` alike?"""
49
- try:
50
- return left.resolve() == right.resolve()
51
- except OSError: # pragma: no cover - unresolvable path
52
- return left.absolute() == right.absolute()
53
-
54
-
55
49
  def _resolve(root: Any, path: str) -> Any:
56
50
  obj = root
57
51
  for part in path.split("/"):
@@ -96,7 +90,7 @@ def recast_store(
96
90
  else None
97
91
  )
98
92
 
99
- if output is not None and not inplace and _same_path(file, output):
93
+ if output is not None and not inplace and same_path(file, output):
100
94
  # Opening the destination "w" clears it while the source is still
101
95
  # being read from it. HDF5 refuses; Zarr does not, and quietly
102
96
  # produced an empty store where the data used to be.
@@ -105,69 +99,54 @@ def recast_store(
105
99
  "it, which writes to a temporary file first."
106
100
  )
107
101
 
108
- if inplace:
109
- backend = detect_backend(file)
110
- if backend == "zarr":
111
- base = file.stem if file.suffix else file.name
112
- dst_path = file.with_name(f"{base}.recast-tmp.zarr")
113
- else:
114
- dst_path = file.with_name(f"{file.name}.recast-tmp")
115
- if dst_path.exists():
116
- raise FileExistsError(f"Temporary path already exists: {dst_path}")
117
- else:
118
- dst_path = output
102
+ dst_path = file if inplace else Path(output)
119
103
 
120
104
  if zarr_format is None and detect_backend(file) == "zarr":
121
105
  with open_store(file, "r") as probe:
122
106
  zarr_format = probe.zarr_format
123
107
 
124
- with open_store(file, "r") as src_store:
125
- src = src_store.root
126
- targets = discover_matrices(src) if convert_all else list(entries or [])
127
- if not targets:
128
- raise ValueError(
129
- "No matrices selected. Name one (e.g. `X`) or pass --all."
130
- )
131
- # Every check first, before the destination exists. A refusal must
132
- # not leave behind a store holding a copy of obs and var and nothing
133
- # else -- the caller cannot tell that from a finished conversion.
134
- plans = {
135
- path: plan_conversion(
136
- _resolve(src, path),
137
- path,
138
- dtype=data_dtype,
139
- index_dtype=index_dtype,
140
- layout=layout,
141
- chunk=chunk,
142
- force=force,
143
- console=console,
108
+ # The source is opened inside the staging block, so it is closed before
109
+ # `--inplace` renames it aside.
110
+ with staged_outputs([dst_path]) as (temp,):
111
+ with open_store(file, "r") as src_store:
112
+ src = src_store.root
113
+ targets = discover_matrices(src) if convert_all else list(entries or [])
114
+ if not targets:
115
+ raise ValueError(
116
+ "No matrices selected. Name one (e.g. `X`) or pass --all."
117
+ )
118
+ # Every check first, before the destination exists. A refusal must
119
+ # not leave behind a store holding a copy of obs and var and nothing
120
+ # else -- the caller cannot tell that from a finished conversion.
121
+ plans = {
122
+ path: plan_conversion(
123
+ _resolve(src, path),
124
+ path,
125
+ dtype=data_dtype,
126
+ index_dtype=index_dtype,
127
+ layout=layout,
128
+ chunk=chunk,
129
+ force=force,
130
+ console=console,
131
+ )
132
+ for path in targets
133
+ }
134
+
135
+ console.print(
136
+ f"[cyan]Converting {len(targets)} matrix/matrices:[/] "
137
+ + ", ".join(targets)
144
138
  )
145
- for path in targets
146
- }
147
-
148
- console.print(
149
- f"[cyan]Converting {len(targets)} matrix/matrices:[/] "
150
- + ", ".join(targets)
151
- )
152
139
 
153
- with open_store(dst_path, "w", zarr_format=zarr_format) as dst_store:
154
- dst = dst_store.root
155
- _write(
156
- src, dst, plans,
157
- chunk=chunk, chunk_rows=chunk_rows, in_memory=in_memory,
158
- console=console,
159
- )
160
- ensure_anndata_skeleton(dst)
140
+ with open_store(temp, "w", zarr_format=zarr_format) as dst_store:
141
+ dst = dst_store.root
142
+ _write(
143
+ src, dst, plans,
144
+ chunk=chunk, chunk_rows=chunk_rows, in_memory=in_memory,
145
+ console=console,
146
+ )
147
+ ensure_anndata_skeleton(dst)
161
148
 
162
149
  if inplace:
163
- if file.is_dir():
164
- shutil.rmtree(file)
165
- elif file.exists():
166
- file.unlink()
167
- if dst_path.is_dir():
168
- shutil.move(str(dst_path), str(file))
169
- else:
170
- dst_path.replace(file)
171
150
  console.print(f"[green]Converted[/] {file}")
172
151
  else:
173
152
  console.print(f"[green]Wrote[/] {dst_path}")
@@ -4,6 +4,11 @@ Requested in issue #2, modelled on cellgeni/scraft's `split_h5ad` but
4
4
  streaming: the source is never loaded into memory, only the column being split
5
5
  on is read, and each output is produced by the same subset machinery used by
6
6
  `adata subset`, fanned out so every matrix is read once for all groups.
7
+
8
+ Every output is written beside its destination and renamed into place only
9
+ once all of them are complete, so a split that fails leaves no partial stores
10
+ (and existing outputs as they were). The manifest is written last, so it
11
+ exists only when everything it lists does.
7
12
  """
8
13
 
9
14
  from __future__ import annotations
@@ -19,6 +24,8 @@ from adata.core.select import group_indices
19
24
  from adata.core.subset import split_h5ad
20
25
  from adata.storage import detect_backend, open_store
21
26
  from adata.util.names import unique_names
27
+ from adata.util.path import same_path
28
+ from adata.util.staging import staged_outputs
22
29
 
23
30
 
24
31
  def split_store(
@@ -78,6 +85,13 @@ def split_store(
78
85
  for label, out_path, count in planned:
79
86
  console.print(f" {label!r} -> {out_path} ({count} {axis})")
80
87
 
88
+ for _, out_path, _ in planned:
89
+ if same_path(file, out_path):
90
+ raise ValueError(
91
+ f"Output path is the input: {out_path}. Choose another "
92
+ "--output-dir."
93
+ )
94
+
81
95
  if dry_run:
82
96
  console.print("[yellow]Dry run: nothing written.[/]")
83
97
  return planned
@@ -86,20 +100,21 @@ def split_store(
86
100
 
87
101
  # One call for every group, so each matrix is read once rather than once
88
102
  # per group.
89
- split_h5ad(
90
- file,
91
- [
92
- (
93
- out_path,
94
- np.sort(groups[label]) if axis == "obs" else None,
95
- np.sort(groups[label]) if axis == "var" else None,
96
- )
97
- for label, out_path, _ in planned
98
- ],
99
- chunk_rows=chunk_rows,
100
- console=console,
101
- zarr_format=zarr_format,
102
- )
103
+ with staged_outputs([out_path for _, out_path, _ in planned]) as temps:
104
+ split_h5ad(
105
+ file,
106
+ [
107
+ (
108
+ temp,
109
+ np.sort(groups[label]) if axis == "obs" else None,
110
+ np.sort(groups[label]) if axis == "var" else None,
111
+ )
112
+ for (label, _, _), temp in zip(planned, temps)
113
+ ],
114
+ chunk_rows=chunk_rows,
115
+ console=console,
116
+ zarr_format=zarr_format,
117
+ )
103
118
 
104
119
  if manifest:
105
120
  _write_manifest(file, output_dir, column, planned, console)
@@ -43,9 +43,11 @@ from adata.storage import (
43
43
  copy_tree,
44
44
  create_dataset,
45
45
  is_dataset,
46
+ Store,
46
47
  is_group,
47
48
  open_store,
48
49
  )
50
+ from adata.util.staging import staged_outputs
49
51
 
50
52
  MERGE_STRATEGIES = ("same", "unique", "first", "only")
51
53
  #: What the CLI accepts. "drop" is the default and maps to no merge at all.
@@ -827,6 +829,17 @@ def _hashable(value: Any) -> Any:
827
829
  return value
828
830
 
829
831
 
832
+ def _inputs_zarr_format(stores: Sequence[Store]) -> Optional[int]:
833
+ """The Zarr version the inputs agree on, or None to take the default.
834
+
835
+ A Zarr output follows its inputs, as every other command's follows its
836
+ source, so v2 inputs are not silently upgraded. h5ad inputs have no
837
+ version and do not count; Zarr inputs of both versions get the default.
838
+ """
839
+ versions = {s.zarr_format for s in stores} - {None}
840
+ return versions.pop() if len(versions) == 1 else None
841
+
842
+
830
843
  def concat_on_disk(
831
844
  files: Sequence[Path],
832
845
  output: Path,
@@ -906,7 +919,14 @@ def concat_on_disk(
906
919
  # half-written store behind.
907
920
  check_matrix_encodings(roots, console)
908
921
 
909
- with open_store(output, "w", zarr_format=zarr_format) as dst_store:
922
+ if zarr_format is None:
923
+ zarr_format = _inputs_zarr_format(stores)
924
+
925
+ # Written beside the output and renamed into place once complete, so
926
+ # a failure part way leaves nothing that reads as a finished store.
927
+ with staged_outputs([output]) as (temp,), open_store(
928
+ temp, "w", zarr_format=zarr_format
929
+ ) as dst_store:
910
930
  dst = dst_store.root
911
931
  _write_obs(
912
932
  dst, roots, target_obs, obs_counts, join, label, input_keys, console
@@ -5,7 +5,6 @@ from __future__ import annotations
5
5
  from contextlib import ExitStack
6
6
  from dataclasses import dataclass
7
7
  from pathlib import Path
8
- import shutil
9
8
  from typing import Optional, Set, Tuple, List, Dict, Any
10
9
 
11
10
  import numpy as np
@@ -43,6 +42,8 @@ from adata.storage import (
43
42
  with_compression,
44
43
  zarr_format_of,
45
44
  )
45
+ from adata.util.path import same_path
46
+ from adata.util.staging import staged_outputs
46
47
 
47
48
 
48
49
  def _target_backend(dst_group: Any) -> str:
@@ -952,18 +953,15 @@ def subset_h5ad(
952
953
  if not inplace and output is None:
953
954
  raise ValueError("Output file is required unless --inplace is specified.")
954
955
 
955
- if inplace:
956
- src_backend = detect_backend(file)
957
- if src_backend == "zarr":
958
- base_name = file.stem if file.suffix else file.name
959
- tmp_path = file.with_name(f"{base_name}.subset-tmp.zarr")
960
- else:
961
- tmp_path = file.with_name(f"{file.name}.subset-tmp")
962
- if tmp_path.exists():
963
- raise FileExistsError(f"Temporary path already exists: {tmp_path}")
964
- dst_path = tmp_path
965
- else:
966
- dst_path = output
956
+ if not inplace and same_path(file, output):
957
+ # Opening the output "w" would empty a Zarr source before a single
958
+ # element had been read from it.
959
+ raise ValueError(
960
+ f"Output path is the input: {output}. Use --inplace to replace "
961
+ "it, which writes to a temporary file first."
962
+ )
963
+
964
+ dst_path = file if inplace else Path(output)
967
965
 
968
966
  if zarr_format is None and detect_backend(file) == "zarr":
969
967
  with open_store(file, "r") as probe:
@@ -975,12 +973,17 @@ def subset_h5ad(
975
973
  status = console.status("[magenta]Opening files...[/]")
976
974
  status.start()
977
975
  try:
978
- with open_store(file, "r") as src_store, open_store(
979
- dst_path, "w", zarr_format=zarr_format
980
- ) as dst_store:
976
+ # The output is written beside its destination and renamed into place
977
+ # only once complete, so a failure leaves any existing output (or,
978
+ # with --inplace, the source) as it was. The source is opened inside
979
+ # the staging block, so it is closed before it is renamed aside.
980
+ with staged_outputs([dst_path]) as (temp,), open_store(
981
+ file, "r"
982
+ ) as src_store:
981
983
  src = src_store.root
982
- dst = dst_store.root
983
984
 
985
+ # The selection is resolved before the output exists: a query
986
+ # that fails must not leave an empty store behind.
984
987
  obs_idx = (
985
988
  obs_indices
986
989
  if obs_indices is not None
@@ -1000,26 +1003,16 @@ def subset_h5ad(
1000
1003
 
1001
1004
  status.stop()
1002
1005
 
1003
- _write_targets(
1004
- src,
1005
- [Target(dst, obs_idx, var_idx, var_keep)],
1006
- chunk_rows=chunk_rows,
1007
- console=console,
1008
- )
1006
+ with open_store(temp, "w", zarr_format=zarr_format) as dst_store:
1007
+ _write_targets(
1008
+ src,
1009
+ [Target(dst_store.root, obs_idx, var_idx, var_keep)],
1010
+ chunk_rows=chunk_rows,
1011
+ console=console,
1012
+ )
1009
1013
  finally:
1010
1014
  status.stop()
1011
1015
 
1012
- if inplace:
1013
- if file.exists():
1014
- if file.is_dir():
1015
- shutil.rmtree(file)
1016
- else:
1017
- file.unlink()
1018
- if dst_path.is_dir():
1019
- shutil.move(str(dst_path), str(file))
1020
- else:
1021
- dst_path.replace(file)
1022
-
1023
1016
 
1024
1017
  def _open_output_limit() -> int:
1025
1018
  """How many output stores `split_h5ad` may hold open at once.
@@ -0,0 +1,19 @@
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+
6
+ def norm_path(path: str) -> str:
7
+ """Normalize object paths used inside h5ad/zarr stores."""
8
+ value = path.strip()
9
+ if not value:
10
+ raise ValueError("Object path must be non-empty.")
11
+ return value.lstrip("/")
12
+
13
+
14
+ def same_path(left: Path, right: Path) -> bool:
15
+ """Do these name the same store, through symlinks and `..` alike?"""
16
+ try:
17
+ return Path(left).resolve() == Path(right).resolve()
18
+ except OSError: # pragma: no cover - unresolvable path
19
+ return Path(left).absolute() == Path(right).absolute()
@@ -0,0 +1,99 @@
1
+ """Write outputs beside their destination, and move them into place at the end.
2
+
3
+ A command that fails half way must not leave a half-written store where the
4
+ output should be: the next command to open it would read whatever got written
5
+ as if it were the whole thing. Writing to a temporary sibling and renaming it
6
+ only on success means the output either appears whole or not at all.
7
+
8
+ An existing output is renamed aside (`out.adata-old.zarr`) before its
9
+ successor is renamed in, and deleted only once every output is in place. If a
10
+ rename fails part way, the outputs already swapped are put back. A run killed
11
+ between the two renames leaves the old data under the `adata-old` name, never
12
+ nowhere, and the next run refuses to start until that has been dealt with.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import shutil
18
+ from contextlib import contextmanager
19
+ from pathlib import Path
20
+ from typing import Iterator, List, Sequence, Tuple
21
+
22
+
23
+ def _sibling(path: Path, tag: str) -> Path:
24
+ """`out.zarr` -> `out.<tag>.zarr`, keeping the suffix that picks the backend."""
25
+ return path.with_name(f"{path.stem}.{tag}{path.suffix}")
26
+
27
+
28
+ def _temp_for(path: Path) -> Path:
29
+ temp = _sibling(path, "adata-tmp")
30
+ if path.is_dir() and path.suffix != ".zarr":
31
+ # A Zarr store named without the suffix. A temporary named the same
32
+ # way would not exist yet, and so be opened as HDF5.
33
+ temp = temp.with_name(f"{temp.name}.zarr")
34
+ return temp
35
+
36
+
37
+ def _remove(path: Path) -> None:
38
+ if path.is_dir() and not path.is_symlink():
39
+ shutil.rmtree(path)
40
+ elif path.exists() or path.is_symlink():
41
+ path.unlink()
42
+
43
+
44
+ def _exists(path: Path) -> bool:
45
+ return path.exists() or path.is_symlink()
46
+
47
+
48
+ @contextmanager
49
+ def staged_outputs(outputs: Sequence[Path]) -> Iterator[List[Path]]:
50
+ """Yield temporary paths to write `outputs` to; move them into place after.
51
+
52
+ On success each output is swapped in: an existing one is set aside first
53
+ and deleted only once every new output is in place. On failure the
54
+ temporary paths are deleted and the outputs are left as they were.
55
+ """
56
+ outputs = [Path(p) for p in outputs]
57
+ temps = [_temp_for(p) for p in outputs]
58
+ backups = [_sibling(p, "adata-old") for p in outputs]
59
+ for leftover in temps + backups:
60
+ if _exists(leftover):
61
+ raise FileExistsError(
62
+ f"'{leftover}' already exists, probably left by an interrupted "
63
+ "run. Check what it holds, remove it, and try again."
64
+ )
65
+
66
+ try:
67
+ yield temps
68
+ except BaseException:
69
+ for temp in temps:
70
+ _remove(temp)
71
+ raise
72
+
73
+ swapped: List[Tuple[Path, Path, bool]] = []
74
+ try:
75
+ for output, temp, backup in zip(outputs, temps, backups):
76
+ had_output = _exists(output)
77
+ if had_output:
78
+ output.rename(backup)
79
+ try:
80
+ temp.rename(output)
81
+ except BaseException:
82
+ if had_output:
83
+ backup.rename(output)
84
+ raise
85
+ swapped.append((output, backup, had_output))
86
+ except BaseException:
87
+ # Put back what was already swapped, newest first, and drop the
88
+ # temporaries that never made it.
89
+ for output, backup, had_output in reversed(swapped):
90
+ _remove(output)
91
+ if had_output:
92
+ backup.rename(output)
93
+ for temp in temps:
94
+ _remove(temp)
95
+ raise
96
+
97
+ for _, backup, had_output in swapped:
98
+ if had_output:
99
+ _remove(backup)
@@ -1,9 +0,0 @@
1
- from __future__ import annotations
2
-
3
-
4
- def norm_path(path: str) -> str:
5
- """Normalize object paths used inside h5ad/zarr stores."""
6
- value = path.strip()
7
- if not value:
8
- raise ValueError("Object path must be non-empty.")
9
- return value.lstrip("/")
@@ -1,63 +0,0 @@
1
- """Write outputs beside their destination, and move them into place at the end.
2
-
3
- A conversion that fails half way must not leave a half-written store where
4
- the output should be: the next command to open it would read whatever got
5
- written as if it were the whole thing. Writing to a temporary sibling and
6
- renaming it only on success means the output either appears whole or not at
7
- all, and an existing output is replaced only once its successor is complete.
8
- """
9
-
10
- from __future__ import annotations
11
-
12
- import shutil
13
- from contextlib import contextmanager
14
- from pathlib import Path
15
- from typing import Iterator, List, Sequence
16
-
17
-
18
- def _sibling(path: Path, tag: str) -> Path:
19
- """`out.zarr` -> `out.<tag>.zarr`, keeping the suffix that picks the backend."""
20
- return path.with_name(f"{path.stem}.{tag}{path.suffix}")
21
-
22
-
23
- def _remove(path: Path) -> None:
24
- if path.is_dir() and not path.is_symlink():
25
- shutil.rmtree(path)
26
- elif path.exists() or path.is_symlink():
27
- path.unlink()
28
-
29
-
30
- @contextmanager
31
- def staged_outputs(outputs: Sequence[Path]) -> Iterator[List[Path]]:
32
- """Yield temporary paths to write `outputs` to; move them into place after.
33
-
34
- On success each output is swapped in: an existing one is set aside first
35
- and deleted only once every new output is in place. On failure the
36
- temporary paths are deleted and the outputs are left as they were.
37
- """
38
- outputs = [Path(p) for p in outputs]
39
- temps = [_sibling(p, "adata-tmp") for p in outputs]
40
- for temp in temps:
41
- if temp.exists():
42
- raise FileExistsError(
43
- f"'{temp}' already exists, probably left by an interrupted run. "
44
- "Remove it and try again."
45
- )
46
-
47
- try:
48
- yield temps
49
- except BaseException:
50
- for temp in temps:
51
- _remove(temp)
52
- raise
53
-
54
- backups = []
55
- for output, temp in zip(outputs, temps):
56
- if output.exists():
57
- backup = _sibling(output, "adata-old")
58
- _remove(backup)
59
- output.rename(backup)
60
- backups.append(backup)
61
- temp.rename(output)
62
- for backup in backups:
63
- _remove(backup)
File without changes
File without changes