pyadata-cli 0.7.0__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/PKG-INFO +4 -3
  2. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/README.md +3 -2
  3. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/pyproject.toml +1 -1
  4. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/cli.py +41 -24
  5. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/convert.py +2 -8
  6. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/import_data.py +3 -1
  7. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/recast.py +52 -73
  8. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/split.py +29 -14
  9. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/conversion/__init__.py +25 -4
  10. pyadata_cli-0.8.0/src/adata/conversion/features.py +380 -0
  11. pyadata_cli-0.8.0/src/adata/conversion/mtx.py +605 -0
  12. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/conversion/tenx.py +41 -275
  13. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/concat.py +21 -1
  14. pyadata_cli-0.8.0/src/adata/core/csr_build.py +328 -0
  15. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/recast.py +19 -102
  16. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/subset.py +27 -34
  17. pyadata_cli-0.8.0/src/adata/formats/mtx.py +503 -0
  18. pyadata_cli-0.8.0/src/adata/formats/sparse.py +152 -0
  19. pyadata_cli-0.8.0/src/adata/formats/textio.py +236 -0
  20. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/storage/__init__.py +2 -10
  21. pyadata_cli-0.8.0/src/adata/util/path.py +31 -0
  22. pyadata_cli-0.8.0/src/adata/util/staging.py +102 -0
  23. pyadata_cli-0.7.0/src/adata/formats/sparse.py +0 -253
  24. pyadata_cli-0.7.0/src/adata/util/path.py +0 -9
  25. pyadata_cli-0.7.0/src/adata/util/staging.py +0 -63
  26. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/LICENSE +0 -0
  27. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/__init__.py +0 -0
  28. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/__init__.py +0 -0
  29. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/concat.py +0 -0
  30. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/create.py +0 -0
  31. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/export.py +0 -0
  32. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/info.py +0 -0
  33. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/ls.py +0 -0
  34. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/subset.py +0 -0
  35. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/conversion/anndata_copy.py +0 -0
  36. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/conversion/base.py +0 -0
  37. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/__init__.py +0 -0
  38. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/info.py +0 -0
  39. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/query.py +0 -0
  40. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/read.py +0 -0
  41. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/select.py +0 -0
  42. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/elements/__init__.py +0 -0
  43. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/elements/read.py +0 -0
  44. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/elements/spec.py +0 -0
  45. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/elements/strings.py +0 -0
  46. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/elements/write.py +0 -0
  47. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/__init__.py +0 -0
  48. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/array.py +0 -0
  49. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/common.py +0 -0
  50. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/dataframe.py +0 -0
  51. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/image.py +0 -0
  52. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/json_data.py +0 -0
  53. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/validate.py +0 -0
  54. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/info.py +0 -0
  55. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/py.typed +0 -0
  56. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/read.py +0 -0
  57. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/util/__init__.py +0 -0
  58. {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/util/names.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pyadata-cli
3
- Version: 0.7.0
3
+ Version: 0.8.0
4
4
  Summary: Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores
5
5
  Keywords: anndata,h5ad,zarr,bioinformatics,single-cell,cli
6
6
  Author: Aljes Binkevich
@@ -48,7 +48,7 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with
48
48
  - Auto-detects `.h5ad` files vs `.zarr` directories
49
49
  - Chunked processing for dense and sparse matrices (CSR/CSC)
50
50
  - Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
51
- - Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices
51
+ - Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices and 10x / STARsolo MTX folders
52
52
  - Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
53
53
 
54
54
  **Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
@@ -96,7 +96,7 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
96
96
  - `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
97
97
  - `split` – write one store per distinct value of an annotation column, with a CSV manifest.
98
98
  - `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
99
- - `convert` – change a store's format: h5ad ↔ Zarr v2/v3, and 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) in and out, streaming, with the output written beside its destination and moved into place only once complete.
99
+ - `convert` – change a store's format: h5ad ↔ Zarr v2/v3, 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) and 10x / STARsolo MTX folders in and out, streaming, with the output written beside its destination and moved into place only once complete.
100
100
  - `recast` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced. Called `convert` until 0.7.0.
101
101
  - `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
102
102
  - `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
@@ -140,6 +140,7 @@ adata convert data.h5ad -o data.zarr
140
140
  adata convert data.zarr -o data.h5ad --compression gzip
141
141
  adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
142
142
  adata convert counts.h5ad -o for_cellbender.h5 --layer counts
143
+ adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
143
144
  ```
144
145
 
145
146
  ## Documentation
@@ -8,7 +8,7 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with
8
8
  - Auto-detects `.h5ad` files vs `.zarr` directories
9
9
  - Chunked processing for dense and sparse matrices (CSR/CSC)
10
10
  - Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
11
- - Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices
11
+ - Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices and 10x / STARsolo MTX folders
12
12
  - Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
13
13
 
14
14
  **Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
@@ -56,7 +56,7 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
56
56
  - `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
57
57
  - `split` – write one store per distinct value of an annotation column, with a CSV manifest.
58
58
  - `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
59
- - `convert` – change a store's format: h5ad ↔ Zarr v2/v3, and 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) in and out, streaming, with the output written beside its destination and moved into place only once complete.
59
+ - `convert` – change a store's format: h5ad ↔ Zarr v2/v3, 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) and 10x / STARsolo MTX folders in and out, streaming, with the output written beside its destination and moved into place only once complete.
60
60
  - `recast` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced. Called `convert` until 0.7.0.
61
61
  - `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
62
62
  - `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
@@ -100,6 +100,7 @@ adata convert data.h5ad -o data.zarr
100
100
  adata convert data.zarr -o data.h5ad --compression gzip
101
101
  adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
102
102
  adata convert counts.h5ad -o for_cellbender.h5 --layer counts
103
+ adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
103
104
  ```
104
105
 
105
106
  ## Documentation
@@ -3,7 +3,7 @@
3
3
  # so the distribution is published as `pyadata-cli`. The import package
4
4
  # and the command are both still `adata`.
5
5
  name = "pyadata-cli"
6
- version = "0.7.0"
6
+ version = "0.8.0"
7
7
  description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores"
8
8
  readme = "README.md"
9
9
  requires-python = ">=3.12"
@@ -347,6 +347,10 @@ def subset(
347
347
  adata subset data.h5ad -o out.h5ad -q "n_counts > 1000 and cluster in A,B"
348
348
  adata subset data.h5ad -o out.h5ad --var-query "highly_variable == True"
349
349
  """
350
+ if zarr_format is not None and zarr_format not in (2, 3):
351
+ console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.")
352
+ raise typer.Exit(code=1)
353
+
350
354
  if obs is None and var is None and obs_query is None and var_query is None:
351
355
  console.print(
352
356
  "[bold red]Error:[/] Provide at least one of --obs, --var, "
@@ -433,7 +437,9 @@ def concat(
433
437
  1024, "--chunk", "-C", help="Row chunk size while streaming"
434
438
  ),
435
439
  zarr_format: Optional[int] = typer.Option(
436
- None, "--zarr-format", help="Zarr spec version to write (2 or 3)"
440
+ None,
441
+ "--zarr-format",
442
+ help="Zarr spec version to write (defaults to the inputs' when they agree)",
437
443
  ),
438
444
  ) -> None:
439
445
  """
@@ -447,6 +453,10 @@ def concat(
447
453
  adata concat *.h5ad -o merged.h5ad --join outer --label sample
448
454
  adata concat a.h5ad b.h5ad -o m.h5ad --keys a,b --index-unique - --uns-merge same
449
455
  """
456
+ if zarr_format is not None and zarr_format not in (2, 3):
457
+ console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.")
458
+ raise typer.Exit(code=1)
459
+
450
460
  for name, value in (("--merge", merge), ("--uns-merge", uns_merge)):
451
461
  if value is not None and value not in MERGE_CHOICES:
452
462
  console.print(
@@ -543,7 +553,7 @@ def convert(
543
553
  ctx: typer.Context,
544
554
  file: Path = typer.Argument(
545
555
  ...,
546
- help="Input file or store",
556
+ help="Input file, store, or 10x/STARsolo matrix folder",
547
557
  exists=True,
548
558
  readable=True,
549
559
  dir_okay=True,
@@ -553,7 +563,8 @@ def convert(
553
563
  None,
554
564
  "--output",
555
565
  "-o",
556
- help="Output path; its extension picks the format (.h5ad, .zarr, .h5)",
566
+ help="Output path; its extension picks the format (.h5ad, .zarr, .h5), "
567
+ "or a folder with --to 10x-mtx",
557
568
  dir_okay=True,
558
569
  file_okay=True,
559
570
  ),
@@ -576,7 +587,7 @@ def convert(
576
587
  None,
577
588
  "--compression",
578
589
  help="none, gzip or lzf for h5ad; none, gzip, zstd or blosc for zarr "
579
- "(defaults to the source's)",
590
+ "(defaults to the source's); gzip or none for a 10x-mtx folder",
580
591
  ),
581
592
  force: bool = typer.Option(
582
593
  False,
@@ -591,7 +602,8 @@ def convert(
591
602
  genome: Optional[str] = typer.Option(
592
603
  None,
593
604
  "--genome",
594
- help="10x input: the genome to keep (picks the group in a Cell Ranger 2 file)",
605
+ help="10x input: the genome to keep (picks the group or subfolder of a "
606
+ "Cell Ranger 2 matrix)",
595
607
  ),
596
608
  var_names: str = typer.Option(
597
609
  "gene_symbols",
@@ -647,11 +659,13 @@ def convert(
647
659
  Convert a store to another format.
648
660
 
649
661
  The output's extension picks the format: .h5ad, .zarr, or .h5 for a 10x
650
- Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5). Between h5ad and
651
- zarr every element is copied, streamed, keeping the source's chunking and
652
- compression where the target can express them. A 10x matrix is read as
653
- scanpy.read_10x_h5(gex_only=False) reads it, keeping integer counts, and
654
- written as Cell Ranger writes it.
662
+ Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5); --to 10x-mtx
663
+ writes a Matrix Market folder. Between h5ad and zarr every element is
664
+ copied, streamed, keeping the source's chunking and compression where the
665
+ target can express them. A 10x matrix is read as scanpy's read_10x_h5 or
666
+ read_10x_mtx (gex_only=False) reads it, keeping integer counts, and
667
+ written as Cell Ranger writes it. MTX folders from Cell Ranger and
668
+ STARsolo, velocyto's included, are read in two streaming passes.
655
669
 
656
670
  Nothing is written until every check has passed, and the output is moved
657
671
  into place only once complete: a failed conversion leaves nothing
@@ -667,6 +681,9 @@ def convert(
667
681
  adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad
668
682
  adata convert raw_feature_bc_matrix.h5 -o rna.h5ad --feature-type "Gene Expression"
669
683
  adata convert counts.h5ad -o for_cellbender.h5 --layer counts
684
+ adata convert Solo.out/Gene/filtered -o counts.h5ad
685
+ adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
686
+ adata convert counts.h5ad -o counts_mtx --to 10x-mtx --layer counts
670
687
  """
671
688
  entries = list(ctx.args)
672
689
  recast_flags = _given(ctx, _RECAST_FLAGS)
@@ -1107,7 +1124,7 @@ def export_sparse(
1107
1124
  "--output",
1108
1125
  "-o",
1109
1126
  writable=True,
1110
- help="Output .mtx file path (defaults to stdout)",
1127
+ help="Output .mtx file path; .mtx.gz gzips it (defaults to stdout)",
1111
1128
  ),
1112
1129
  head: Optional[int] = typer.Option(
1113
1130
  None, "--head", "-n", help="Output only the first n entries of mtx file"
@@ -1129,9 +1146,9 @@ def export_sparse(
1129
1146
  Export a sparse matrix (CSR/CSC) to Matrix Market (.mtx) format.
1130
1147
 
1131
1148
  Examples:
1132
- h5ad export sparse data.h5ad X matrix.mtx
1133
- h5ad export sparse data.h5ad layers/counts counts.mtx
1134
- h5ad export sparse data.h5ad X --head 100
1149
+ adata export sparse data.h5ad X -o matrix.mtx
1150
+ adata export sparse data.h5ad layers/counts -o counts.mtx
1151
+ adata export sparse data.h5ad X --head 100
1135
1152
  """
1136
1153
 
1137
1154
  try:
@@ -1179,8 +1196,8 @@ def export_dict(
1179
1196
  Export a dict/group or scalar to JSON format.
1180
1197
 
1181
1198
  Examples:
1182
- h5ad export dict data.h5ad uns metadata.json
1183
- h5ad export dict data.h5ad uns/colors colors.json
1199
+ adata export dict data.h5ad uns metadata.json
1200
+ adata export dict data.h5ad uns/colors colors.json
1184
1201
  """
1185
1202
 
1186
1203
  try:
@@ -1219,7 +1236,7 @@ def export_image(
1219
1236
  The array should be 2D (H,W) or 3D (H,W,C) with C in {1,3,4}.
1220
1237
 
1221
1238
  Examples:
1222
- h5ad export image data.h5ad uns/spatial/image tissue.png
1239
+ adata export image data.h5ad uns/spatial/image -o tissue.png
1223
1240
  """
1224
1241
 
1225
1242
  try:
@@ -1363,8 +1380,8 @@ def import_array(
1363
1380
  Dimensions are validated against existing obs/var.
1364
1381
 
1365
1382
  Examples:
1366
- h5ad import array data.h5ad obsm/X_pca pca.npy -o output.h5ad
1367
- h5ad import array data.h5ad X matrix.npy --inplace
1383
+ adata import array data.h5ad obsm/X_pca pca.npy -o output.h5ad
1384
+ adata import array data.h5ad X matrix.npy --inplace
1368
1385
  """
1369
1386
  from adata.commands.import_data import _import_npy
1370
1387
 
@@ -1397,7 +1414,7 @@ def import_sparse(
1397
1414
  ..., help="Object path to create/replace (e.g., 'X', 'layers/counts')"
1398
1415
  ),
1399
1416
  input_file: Path = typer.Argument(
1400
- ..., help="Input .mtx file", exists=True, readable=True
1417
+ ..., help="Input .mtx or .mtx.gz file", exists=True, readable=True
1401
1418
  ),
1402
1419
  output: Optional[Path] = typer.Option(
1403
1420
  None,
@@ -1419,8 +1436,8 @@ def import_sparse(
1419
1436
  Dimensions are validated against existing obs/var.
1420
1437
 
1421
1438
  Examples:
1422
- h5ad import sparse data.h5ad X matrix.mtx -o output.h5ad
1423
- h5ad import sparse data.h5ad layers/counts counts.mtx --inplace
1439
+ adata import sparse data.h5ad X matrix.mtx -o output.h5ad
1440
+ adata import sparse data.h5ad layers/counts counts.mtx --inplace
1424
1441
  """
1425
1442
  from adata.commands.import_data import _import_mtx
1426
1443
 
@@ -1473,8 +1490,8 @@ def import_dict(
1473
1490
  Import a JSON file into uns or other dict-like groups.
1474
1491
 
1475
1492
  Examples:
1476
- h5ad import dict data.h5ad uns/metadata config.json -o output.h5ad
1477
- h5ad import dict data.h5ad uns settings.json --inplace
1493
+ adata import dict data.h5ad uns/metadata config.json -o output.h5ad
1494
+ adata import dict data.h5ad uns settings.json --inplace
1478
1495
  """
1479
1496
  from adata.commands.import_data import _import_json
1480
1497
 
@@ -20,16 +20,10 @@ from adata.conversion import (
20
20
  detect_format,
21
21
  output_format,
22
22
  )
23
+ from adata.util.path import same_path
23
24
  from adata.util.staging import staged_outputs
24
25
 
25
26
 
26
- def _same_path(left: Path, right: Path) -> bool:
27
- try:
28
- return left.resolve() == right.resolve()
29
- except OSError:
30
- return False
31
-
32
-
33
27
  def convert_store(
34
28
  file: Path,
35
29
  output: Path,
@@ -56,7 +50,7 @@ def convert_store(
56
50
 
57
51
  plan = converter.plan(file, output, src_format, dst_format, options)
58
52
  for out in plan.outputs:
59
- if _same_path(file, out):
53
+ if same_path(file, out):
60
54
  # Writing a Zarr store over its own source empties it before a
61
55
  # single element has been read.
62
56
  raise ValueError(f"Output path is the input: {out}")
@@ -60,7 +60,9 @@ def import_object(
60
60
  console: Console,
61
61
  ) -> None:
62
62
  target_file = _prepare_target_path(file, output_file, inplace, console)
63
- ext = input_file.suffix.lower()
63
+ suffixes = [x.lower() for x in input_file.suffixes]
64
+ # A gzipped matrix is still a matrix: `counts.mtx.gz` imports as `.mtx`.
65
+ ext = suffixes[-2] if len(suffixes) > 1 and suffixes[-1] == ".gz" else input_file.suffix.lower()
64
66
 
65
67
  if ext not in EXTENSION_FORMAT:
66
68
  raise ValueError(
@@ -2,13 +2,14 @@
2
2
 
3
3
  Writes a whole new store rather than editing one in place, for the same
4
4
  reason `subset` does: a conversion that fails half way leaves the original
5
- untouched, and `--inplace` becomes an atomic swap of a finished file rather
6
- than a partial edit of a live one.
5
+ untouched. The new store is written beside its destination and renamed into
6
+ place only once complete (see `adata.util.staging`), so `--inplace` replaces a
7
+ finished file rather than editing a live one, and the original is deleted only
8
+ after its successor is in place.
7
9
  """
8
10
 
9
11
  from __future__ import annotations
10
12
 
11
- import shutil
12
13
  from pathlib import Path
13
14
  from typing import Any, List, Optional
14
15
 
@@ -25,15 +26,16 @@ from adata.core.recast import (
25
26
  from adata.elements import spec
26
27
  from adata.elements.write import ensure_anndata_skeleton
27
28
  from adata.storage import copy_tree, detect_backend, is_group, open_store
28
-
29
- #: What `--all` means. The matrices an AnnData object is built from, not
30
- #: every 2-D array in the file: obsm/varm hold embeddings, whose dtype is
31
- #: rarely what anyone is trying to shrink. Those still convert by path.
32
- ALL_PREFIXES = ("X", "layers", "raw/X")
33
-
29
+ from adata.util.path import same_path
30
+ from adata.util.staging import staged_outputs
34
31
 
35
32
  def discover_matrices(root: Any) -> List[str]:
36
- """Entry paths `--all` resolves to, in a stable order."""
33
+ """Entry paths `--all` resolves to, in a stable order.
34
+
35
+ The matrices an AnnData object is built from, not every 2-D array in the
36
+ file: obsm/varm hold embeddings, whose dtype is rarely what anyone is
37
+ trying to shrink. Those still convert by path.
38
+ """
37
39
  found: List[str] = []
38
40
  if "X" in root:
39
41
  found.append("X")
@@ -44,14 +46,6 @@ def discover_matrices(root: Any) -> List[str]:
44
46
  return found
45
47
 
46
48
 
47
- def _same_path(left: Path, right: Path) -> bool:
48
- """Do these name the same store, through symlinks and `..` alike?"""
49
- try:
50
- return left.resolve() == right.resolve()
51
- except OSError: # pragma: no cover - unresolvable path
52
- return left.absolute() == right.absolute()
53
-
54
-
55
49
  def _resolve(root: Any, path: str) -> Any:
56
50
  obj = root
57
51
  for part in path.split("/"):
@@ -96,7 +90,7 @@ def recast_store(
96
90
  else None
97
91
  )
98
92
 
99
- if output is not None and not inplace and _same_path(file, output):
93
+ if output is not None and not inplace and same_path(file, output):
100
94
  # Opening the destination "w" clears it while the source is still
101
95
  # being read from it. HDF5 refuses; Zarr does not, and quietly
102
96
  # produced an empty store where the data used to be.
@@ -105,69 +99,54 @@ def recast_store(
105
99
  "it, which writes to a temporary file first."
106
100
  )
107
101
 
108
- if inplace:
109
- backend = detect_backend(file)
110
- if backend == "zarr":
111
- base = file.stem if file.suffix else file.name
112
- dst_path = file.with_name(f"{base}.recast-tmp.zarr")
113
- else:
114
- dst_path = file.with_name(f"{file.name}.recast-tmp")
115
- if dst_path.exists():
116
- raise FileExistsError(f"Temporary path already exists: {dst_path}")
117
- else:
118
- dst_path = output
102
+ dst_path = file if inplace else Path(output)
119
103
 
120
104
  if zarr_format is None and detect_backend(file) == "zarr":
121
105
  with open_store(file, "r") as probe:
122
106
  zarr_format = probe.zarr_format
123
107
 
124
- with open_store(file, "r") as src_store:
125
- src = src_store.root
126
- targets = discover_matrices(src) if convert_all else list(entries or [])
127
- if not targets:
128
- raise ValueError(
129
- "No matrices selected. Name one (e.g. `X`) or pass --all."
130
- )
131
- # Every check first, before the destination exists. A refusal must
132
- # not leave behind a store holding a copy of obs and var and nothing
133
- # else -- the caller cannot tell that from a finished conversion.
134
- plans = {
135
- path: plan_conversion(
136
- _resolve(src, path),
137
- path,
138
- dtype=data_dtype,
139
- index_dtype=index_dtype,
140
- layout=layout,
141
- chunk=chunk,
142
- force=force,
143
- console=console,
108
+ # The source is opened inside the staging block, so it is closed before
109
+ # `--inplace` renames it aside.
110
+ with staged_outputs([dst_path]) as (temp,):
111
+ with open_store(file, "r") as src_store:
112
+ src = src_store.root
113
+ targets = discover_matrices(src) if convert_all else list(entries or [])
114
+ if not targets:
115
+ raise ValueError(
116
+ "No matrices selected. Name one (e.g. `X`) or pass --all."
117
+ )
118
+ # Every check first, before the destination exists. A refusal must
119
+ # not leave behind a store holding a copy of obs and var and nothing
120
+ # else -- the caller cannot tell that from a finished conversion.
121
+ plans = {
122
+ path: plan_conversion(
123
+ _resolve(src, path),
124
+ path,
125
+ dtype=data_dtype,
126
+ index_dtype=index_dtype,
127
+ layout=layout,
128
+ chunk=chunk,
129
+ force=force,
130
+ console=console,
131
+ )
132
+ for path in targets
133
+ }
134
+
135
+ console.print(
136
+ f"[cyan]Converting {len(targets)} matrix/matrices:[/] "
137
+ + ", ".join(targets)
144
138
  )
145
- for path in targets
146
- }
147
-
148
- console.print(
149
- f"[cyan]Converting {len(targets)} matrix/matrices:[/] "
150
- + ", ".join(targets)
151
- )
152
139
 
153
- with open_store(dst_path, "w", zarr_format=zarr_format) as dst_store:
154
- dst = dst_store.root
155
- _write(
156
- src, dst, plans,
157
- chunk=chunk, chunk_rows=chunk_rows, in_memory=in_memory,
158
- console=console,
159
- )
160
- ensure_anndata_skeleton(dst)
140
+ with open_store(temp, "w", zarr_format=zarr_format) as dst_store:
141
+ dst = dst_store.root
142
+ _write(
143
+ src, dst, plans,
144
+ chunk=chunk, chunk_rows=chunk_rows, in_memory=in_memory,
145
+ console=console,
146
+ )
147
+ ensure_anndata_skeleton(dst)
161
148
 
162
149
  if inplace:
163
- if file.is_dir():
164
- shutil.rmtree(file)
165
- elif file.exists():
166
- file.unlink()
167
- if dst_path.is_dir():
168
- shutil.move(str(dst_path), str(file))
169
- else:
170
- dst_path.replace(file)
171
150
  console.print(f"[green]Converted[/] {file}")
172
151
  else:
173
152
  console.print(f"[green]Wrote[/] {dst_path}")
@@ -4,6 +4,11 @@ Requested in issue #2, modelled on cellgeni/scraft's `split_h5ad` but
4
4
  streaming: the source is never loaded into memory, only the column being split
5
5
  on is read, and each output is produced by the same subset machinery used by
6
6
  `adata subset`, fanned out so every matrix is read once for all groups.
7
+
8
+ Every output is written beside its destination and renamed into place only
9
+ once all of them are complete, so a split that fails leaves no partial stores
10
+ (and existing outputs as they were). The manifest is written last, so it
11
+ exists only when everything it lists does.
7
12
  """
8
13
 
9
14
  from __future__ import annotations
@@ -19,6 +24,8 @@ from adata.core.select import group_indices
19
24
  from adata.core.subset import split_h5ad
20
25
  from adata.storage import detect_backend, open_store
21
26
  from adata.util.names import unique_names
27
+ from adata.util.path import same_path
28
+ from adata.util.staging import staged_outputs
22
29
 
23
30
 
24
31
  def split_store(
@@ -78,6 +85,13 @@ def split_store(
78
85
  for label, out_path, count in planned:
79
86
  console.print(f" {label!r} -> {out_path} ({count} {axis})")
80
87
 
88
+ for _, out_path, _ in planned:
89
+ if same_path(file, out_path):
90
+ raise ValueError(
91
+ f"Output path is the input: {out_path}. Choose another "
92
+ "--output-dir."
93
+ )
94
+
81
95
  if dry_run:
82
96
  console.print("[yellow]Dry run: nothing written.[/]")
83
97
  return planned
@@ -86,20 +100,21 @@ def split_store(
86
100
 
87
101
  # One call for every group, so each matrix is read once rather than once
88
102
  # per group.
89
- split_h5ad(
90
- file,
91
- [
92
- (
93
- out_path,
94
- np.sort(groups[label]) if axis == "obs" else None,
95
- np.sort(groups[label]) if axis == "var" else None,
96
- )
97
- for label, out_path, _ in planned
98
- ],
99
- chunk_rows=chunk_rows,
100
- console=console,
101
- zarr_format=zarr_format,
102
- )
103
+ with staged_outputs([out_path for _, out_path, _ in planned]) as temps:
104
+ split_h5ad(
105
+ file,
106
+ [
107
+ (
108
+ temp,
109
+ np.sort(groups[label]) if axis == "obs" else None,
110
+ np.sort(groups[label]) if axis == "var" else None,
111
+ )
112
+ for (label, _, _), temp in zip(planned, temps)
113
+ ],
114
+ chunk_rows=chunk_rows,
115
+ console=console,
116
+ zarr_format=zarr_format,
117
+ )
103
118
 
104
119
  if manifest:
105
120
  _write_manifest(file, output_dir, column, planned, console)
@@ -17,7 +17,7 @@ from typing import Dict, FrozenSet, List, Optional
17
17
 
18
18
  import h5py
19
19
 
20
- from adata.conversion import tenx
20
+ from adata.conversion import mtx, tenx
21
21
  from adata.conversion.anndata_copy import AnnDataCopy
22
22
  from adata.conversion.base import ConversionPlan, Converter, Format, Options
23
23
  from adata.storage import has_valid_anndata_root_attrs, is_zarr_path
@@ -74,6 +74,15 @@ FORMATS: Dict[str, Format] = {
74
74
  importer=tenx.TenxImport(),
75
75
  exporter=tenx.TenxExport(),
76
76
  ),
77
+ # A folder, so no extension: chosen as output with `--to 10x-mtx`.
78
+ "10x-mtx": Format(
79
+ name="10x-mtx",
80
+ description="10x Genomics / STARsolo Matrix Market folder",
81
+ extensions=(),
82
+ sniff=mtx.sniff,
83
+ importer=mtx.MtxImport(),
84
+ exporter=mtx.MtxExport(),
85
+ ),
77
86
  }
78
87
 
79
88
  _ANNDATA_COPY = AnnDataCopy()
@@ -106,6 +115,12 @@ def detect_format(path: Path) -> str:
106
115
  for fmt in FORMATS.values():
107
116
  if (fmt.anndata or fmt.sniff is None) and suffix in fmt.extensions:
108
117
  return fmt.name
118
+ if suffix in (".mtx", ".gz") and path.name.lower().endswith((".mtx", ".mtx.gz")):
119
+ raise ValueError(
120
+ f"'{path.name}' is a single Matrix Market file, which `convert` "
121
+ "does not read on its own: give the folder holding it with its "
122
+ "barcodes and features, or use `adata import sparse`."
123
+ )
109
124
  raise ValueError(
110
125
  f"Cannot tell what format '{path}' is. Pass --from, one of: {_names()}."
111
126
  )
@@ -130,15 +145,21 @@ def output_format(path: Path, requested: Optional[str] = None) -> str:
130
145
  if by_extension is None:
131
146
  raise ValueError(
132
147
  f"Cannot tell the output format from '{path.name}'. "
133
- f"Use a known extension or pass --to, one of: {_names()}."
148
+ f"Use a known extension or pass --to, one of: {_names()} "
149
+ "(--to 10x-mtx for a folder)."
134
150
  )
135
151
  return by_extension
136
152
 
137
153
  if by_extension is not None and by_extension != requested:
154
+ extensions = FORMATS[requested].extensions
138
155
  raise ValueError(
139
156
  f"--to {requested} contradicts the output's extension, which means "
140
- f"{by_extension}. Name the output with one of: "
141
- f"{', '.join(FORMATS[requested].extensions)}."
157
+ f"{by_extension}. Name the output "
158
+ + (
159
+ f"with one of: {', '.join(extensions)}."
160
+ if extensions
161
+ else "as a folder, without an extension."
162
+ )
142
163
  )
143
164
  if (requested == "zarr") != (suffix == ".zarr"):
144
165
  raise ValueError(