pyadata-cli 0.7.0__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/PKG-INFO +4 -3
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/README.md +3 -2
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/pyproject.toml +1 -1
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/cli.py +41 -24
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/convert.py +2 -8
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/import_data.py +3 -1
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/recast.py +52 -73
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/split.py +29 -14
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/conversion/__init__.py +25 -4
- pyadata_cli-0.8.0/src/adata/conversion/features.py +380 -0
- pyadata_cli-0.8.0/src/adata/conversion/mtx.py +605 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/conversion/tenx.py +41 -275
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/concat.py +21 -1
- pyadata_cli-0.8.0/src/adata/core/csr_build.py +328 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/recast.py +19 -102
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/subset.py +27 -34
- pyadata_cli-0.8.0/src/adata/formats/mtx.py +503 -0
- pyadata_cli-0.8.0/src/adata/formats/sparse.py +152 -0
- pyadata_cli-0.8.0/src/adata/formats/textio.py +236 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/storage/__init__.py +2 -10
- pyadata_cli-0.8.0/src/adata/util/path.py +31 -0
- pyadata_cli-0.8.0/src/adata/util/staging.py +102 -0
- pyadata_cli-0.7.0/src/adata/formats/sparse.py +0 -253
- pyadata_cli-0.7.0/src/adata/util/path.py +0 -9
- pyadata_cli-0.7.0/src/adata/util/staging.py +0 -63
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/LICENSE +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/concat.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/create.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/export.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/info.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/ls.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/commands/subset.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/conversion/anndata_copy.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/conversion/base.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/info.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/query.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/read.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/core/select.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/elements/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/elements/read.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/elements/spec.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/elements/strings.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/elements/write.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/array.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/common.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/dataframe.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/image.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/json_data.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/formats/validate.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/info.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/py.typed +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/read.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/util/__init__.py +0 -0
- {pyadata_cli-0.7.0 → pyadata_cli-0.8.0}/src/adata/util/names.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pyadata-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores
|
|
5
5
|
Keywords: anndata,h5ad,zarr,bioinformatics,single-cell,cli
|
|
6
6
|
Author: Aljes Binkevich
|
|
@@ -48,7 +48,7 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with
|
|
|
48
48
|
- Auto-detects `.h5ad` files vs `.zarr` directories
|
|
49
49
|
- Chunked processing for dense and sparse matrices (CSR/CSC)
|
|
50
50
|
- Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
|
|
51
|
-
- Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices
|
|
51
|
+
- Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices and 10x / STARsolo MTX folders
|
|
52
52
|
- Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
|
|
53
53
|
|
|
54
54
|
**Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
|
|
@@ -96,7 +96,7 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
|
|
|
96
96
|
- `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
|
|
97
97
|
- `split` – write one store per distinct value of an annotation column, with a CSV manifest.
|
|
98
98
|
- `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
|
|
99
|
-
- `convert` – change a store's format: h5ad ↔ Zarr v2/v3,
|
|
99
|
+
- `convert` – change a store's format: h5ad ↔ Zarr v2/v3, 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) and 10x / STARsolo MTX folders in and out, streaming, with the output written beside its destination and moved into place only once complete.
|
|
100
100
|
- `recast` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced. Called `convert` until 0.7.0.
|
|
101
101
|
- `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
|
|
102
102
|
- `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
|
|
@@ -140,6 +140,7 @@ adata convert data.h5ad -o data.zarr
|
|
|
140
140
|
adata convert data.zarr -o data.h5ad --compression gzip
|
|
141
141
|
adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
|
|
142
142
|
adata convert counts.h5ad -o for_cellbender.h5 --layer counts
|
|
143
|
+
adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
|
|
143
144
|
```
|
|
144
145
|
|
|
145
146
|
## Documentation
|
|
@@ -8,7 +8,7 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with
|
|
|
8
8
|
- Auto-detects `.h5ad` files vs `.zarr` directories
|
|
9
9
|
- Chunked processing for dense and sparse matrices (CSR/CSC)
|
|
10
10
|
- Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
|
|
11
|
-
- Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices
|
|
11
|
+
- Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices and 10x / STARsolo MTX folders
|
|
12
12
|
- Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
|
|
13
13
|
|
|
14
14
|
**Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
|
|
@@ -56,7 +56,7 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
|
|
|
56
56
|
- `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
|
|
57
57
|
- `split` – write one store per distinct value of an annotation column, with a CSV manifest.
|
|
58
58
|
- `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
|
|
59
|
-
- `convert` – change a store's format: h5ad ↔ Zarr v2/v3,
|
|
59
|
+
- `convert` – change a store's format: h5ad ↔ Zarr v2/v3, 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) and 10x / STARsolo MTX folders in and out, streaming, with the output written beside its destination and moved into place only once complete.
|
|
60
60
|
- `recast` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced. Called `convert` until 0.7.0.
|
|
61
61
|
- `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
|
|
62
62
|
- `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
|
|
@@ -100,6 +100,7 @@ adata convert data.h5ad -o data.zarr
|
|
|
100
100
|
adata convert data.zarr -o data.h5ad --compression gzip
|
|
101
101
|
adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
|
|
102
102
|
adata convert counts.h5ad -o for_cellbender.h5 --layer counts
|
|
103
|
+
adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
|
|
103
104
|
```
|
|
104
105
|
|
|
105
106
|
## Documentation
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# so the distribution is published as `pyadata-cli`. The import package
|
|
4
4
|
# and the command are both still `adata`.
|
|
5
5
|
name = "pyadata-cli"
|
|
6
|
-
version = "0.
|
|
6
|
+
version = "0.8.0"
|
|
7
7
|
description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores"
|
|
8
8
|
readme = "README.md"
|
|
9
9
|
requires-python = ">=3.12"
|
|
@@ -347,6 +347,10 @@ def subset(
|
|
|
347
347
|
adata subset data.h5ad -o out.h5ad -q "n_counts > 1000 and cluster in A,B"
|
|
348
348
|
adata subset data.h5ad -o out.h5ad --var-query "highly_variable == True"
|
|
349
349
|
"""
|
|
350
|
+
if zarr_format is not None and zarr_format not in (2, 3):
|
|
351
|
+
console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.")
|
|
352
|
+
raise typer.Exit(code=1)
|
|
353
|
+
|
|
350
354
|
if obs is None and var is None and obs_query is None and var_query is None:
|
|
351
355
|
console.print(
|
|
352
356
|
"[bold red]Error:[/] Provide at least one of --obs, --var, "
|
|
@@ -433,7 +437,9 @@ def concat(
|
|
|
433
437
|
1024, "--chunk", "-C", help="Row chunk size while streaming"
|
|
434
438
|
),
|
|
435
439
|
zarr_format: Optional[int] = typer.Option(
|
|
436
|
-
None,
|
|
440
|
+
None,
|
|
441
|
+
"--zarr-format",
|
|
442
|
+
help="Zarr spec version to write (defaults to the inputs' when they agree)",
|
|
437
443
|
),
|
|
438
444
|
) -> None:
|
|
439
445
|
"""
|
|
@@ -447,6 +453,10 @@ def concat(
|
|
|
447
453
|
adata concat *.h5ad -o merged.h5ad --join outer --label sample
|
|
448
454
|
adata concat a.h5ad b.h5ad -o m.h5ad --keys a,b --index-unique - --uns-merge same
|
|
449
455
|
"""
|
|
456
|
+
if zarr_format is not None and zarr_format not in (2, 3):
|
|
457
|
+
console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.")
|
|
458
|
+
raise typer.Exit(code=1)
|
|
459
|
+
|
|
450
460
|
for name, value in (("--merge", merge), ("--uns-merge", uns_merge)):
|
|
451
461
|
if value is not None and value not in MERGE_CHOICES:
|
|
452
462
|
console.print(
|
|
@@ -543,7 +553,7 @@ def convert(
|
|
|
543
553
|
ctx: typer.Context,
|
|
544
554
|
file: Path = typer.Argument(
|
|
545
555
|
...,
|
|
546
|
-
help="Input file or
|
|
556
|
+
help="Input file, store, or 10x/STARsolo matrix folder",
|
|
547
557
|
exists=True,
|
|
548
558
|
readable=True,
|
|
549
559
|
dir_okay=True,
|
|
@@ -553,7 +563,8 @@ def convert(
|
|
|
553
563
|
None,
|
|
554
564
|
"--output",
|
|
555
565
|
"-o",
|
|
556
|
-
help="Output path; its extension picks the format (.h5ad, .zarr, .h5)"
|
|
566
|
+
help="Output path; its extension picks the format (.h5ad, .zarr, .h5), "
|
|
567
|
+
"or a folder with --to 10x-mtx",
|
|
557
568
|
dir_okay=True,
|
|
558
569
|
file_okay=True,
|
|
559
570
|
),
|
|
@@ -576,7 +587,7 @@ def convert(
|
|
|
576
587
|
None,
|
|
577
588
|
"--compression",
|
|
578
589
|
help="none, gzip or lzf for h5ad; none, gzip, zstd or blosc for zarr "
|
|
579
|
-
"(defaults to the source's)",
|
|
590
|
+
"(defaults to the source's); gzip or none for a 10x-mtx folder",
|
|
580
591
|
),
|
|
581
592
|
force: bool = typer.Option(
|
|
582
593
|
False,
|
|
@@ -591,7 +602,8 @@ def convert(
|
|
|
591
602
|
genome: Optional[str] = typer.Option(
|
|
592
603
|
None,
|
|
593
604
|
"--genome",
|
|
594
|
-
help="10x input: the genome to keep (picks the group
|
|
605
|
+
help="10x input: the genome to keep (picks the group or subfolder of a "
|
|
606
|
+
"Cell Ranger 2 matrix)",
|
|
595
607
|
),
|
|
596
608
|
var_names: str = typer.Option(
|
|
597
609
|
"gene_symbols",
|
|
@@ -647,11 +659,13 @@ def convert(
|
|
|
647
659
|
Convert a store to another format.
|
|
648
660
|
|
|
649
661
|
The output's extension picks the format: .h5ad, .zarr, or .h5 for a 10x
|
|
650
|
-
Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5)
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
662
|
+
Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5); --to 10x-mtx
|
|
663
|
+
writes a Matrix Market folder. Between h5ad and zarr every element is
|
|
664
|
+
copied, streamed, keeping the source's chunking and compression where the
|
|
665
|
+
target can express them. A 10x matrix is read as scanpy's read_10x_h5 or
|
|
666
|
+
read_10x_mtx (gex_only=False) reads it, keeping integer counts, and
|
|
667
|
+
written as Cell Ranger writes it. MTX folders from Cell Ranger and
|
|
668
|
+
STARsolo, velocyto's included, are read in two streaming passes.
|
|
655
669
|
|
|
656
670
|
Nothing is written until every check has passed, and the output is moved
|
|
657
671
|
into place only once complete: a failed conversion leaves nothing
|
|
@@ -667,6 +681,9 @@ def convert(
|
|
|
667
681
|
adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad
|
|
668
682
|
adata convert raw_feature_bc_matrix.h5 -o rna.h5ad --feature-type "Gene Expression"
|
|
669
683
|
adata convert counts.h5ad -o for_cellbender.h5 --layer counts
|
|
684
|
+
adata convert Solo.out/Gene/filtered -o counts.h5ad
|
|
685
|
+
adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
|
|
686
|
+
adata convert counts.h5ad -o counts_mtx --to 10x-mtx --layer counts
|
|
670
687
|
"""
|
|
671
688
|
entries = list(ctx.args)
|
|
672
689
|
recast_flags = _given(ctx, _RECAST_FLAGS)
|
|
@@ -1107,7 +1124,7 @@ def export_sparse(
|
|
|
1107
1124
|
"--output",
|
|
1108
1125
|
"-o",
|
|
1109
1126
|
writable=True,
|
|
1110
|
-
help="Output .mtx file path (defaults to stdout)",
|
|
1127
|
+
help="Output .mtx file path; .mtx.gz gzips it (defaults to stdout)",
|
|
1111
1128
|
),
|
|
1112
1129
|
head: Optional[int] = typer.Option(
|
|
1113
1130
|
None, "--head", "-n", help="Output only the first n entries of mtx file"
|
|
@@ -1129,9 +1146,9 @@ def export_sparse(
|
|
|
1129
1146
|
Export a sparse matrix (CSR/CSC) to Matrix Market (.mtx) format.
|
|
1130
1147
|
|
|
1131
1148
|
Examples:
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1149
|
+
adata export sparse data.h5ad X -o matrix.mtx
|
|
1150
|
+
adata export sparse data.h5ad layers/counts -o counts.mtx
|
|
1151
|
+
adata export sparse data.h5ad X --head 100
|
|
1135
1152
|
"""
|
|
1136
1153
|
|
|
1137
1154
|
try:
|
|
@@ -1179,8 +1196,8 @@ def export_dict(
|
|
|
1179
1196
|
Export a dict/group or scalar to JSON format.
|
|
1180
1197
|
|
|
1181
1198
|
Examples:
|
|
1182
|
-
|
|
1183
|
-
|
|
1199
|
+
adata export dict data.h5ad uns metadata.json
|
|
1200
|
+
adata export dict data.h5ad uns/colors colors.json
|
|
1184
1201
|
"""
|
|
1185
1202
|
|
|
1186
1203
|
try:
|
|
@@ -1219,7 +1236,7 @@ def export_image(
|
|
|
1219
1236
|
The array should be 2D (H,W) or 3D (H,W,C) with C in {1,3,4}.
|
|
1220
1237
|
|
|
1221
1238
|
Examples:
|
|
1222
|
-
|
|
1239
|
+
adata export image data.h5ad uns/spatial/image -o tissue.png
|
|
1223
1240
|
"""
|
|
1224
1241
|
|
|
1225
1242
|
try:
|
|
@@ -1363,8 +1380,8 @@ def import_array(
|
|
|
1363
1380
|
Dimensions are validated against existing obs/var.
|
|
1364
1381
|
|
|
1365
1382
|
Examples:
|
|
1366
|
-
|
|
1367
|
-
|
|
1383
|
+
adata import array data.h5ad obsm/X_pca pca.npy -o output.h5ad
|
|
1384
|
+
adata import array data.h5ad X matrix.npy --inplace
|
|
1368
1385
|
"""
|
|
1369
1386
|
from adata.commands.import_data import _import_npy
|
|
1370
1387
|
|
|
@@ -1397,7 +1414,7 @@ def import_sparse(
|
|
|
1397
1414
|
..., help="Object path to create/replace (e.g., 'X', 'layers/counts')"
|
|
1398
1415
|
),
|
|
1399
1416
|
input_file: Path = typer.Argument(
|
|
1400
|
-
..., help="Input .mtx file", exists=True, readable=True
|
|
1417
|
+
..., help="Input .mtx or .mtx.gz file", exists=True, readable=True
|
|
1401
1418
|
),
|
|
1402
1419
|
output: Optional[Path] = typer.Option(
|
|
1403
1420
|
None,
|
|
@@ -1419,8 +1436,8 @@ def import_sparse(
|
|
|
1419
1436
|
Dimensions are validated against existing obs/var.
|
|
1420
1437
|
|
|
1421
1438
|
Examples:
|
|
1422
|
-
|
|
1423
|
-
|
|
1439
|
+
adata import sparse data.h5ad X matrix.mtx -o output.h5ad
|
|
1440
|
+
adata import sparse data.h5ad layers/counts counts.mtx --inplace
|
|
1424
1441
|
"""
|
|
1425
1442
|
from adata.commands.import_data import _import_mtx
|
|
1426
1443
|
|
|
@@ -1473,8 +1490,8 @@ def import_dict(
|
|
|
1473
1490
|
Import a JSON file into uns or other dict-like groups.
|
|
1474
1491
|
|
|
1475
1492
|
Examples:
|
|
1476
|
-
|
|
1477
|
-
|
|
1493
|
+
adata import dict data.h5ad uns/metadata config.json -o output.h5ad
|
|
1494
|
+
adata import dict data.h5ad uns settings.json --inplace
|
|
1478
1495
|
"""
|
|
1479
1496
|
from adata.commands.import_data import _import_json
|
|
1480
1497
|
|
|
@@ -20,16 +20,10 @@ from adata.conversion import (
|
|
|
20
20
|
detect_format,
|
|
21
21
|
output_format,
|
|
22
22
|
)
|
|
23
|
+
from adata.util.path import same_path
|
|
23
24
|
from adata.util.staging import staged_outputs
|
|
24
25
|
|
|
25
26
|
|
|
26
|
-
def _same_path(left: Path, right: Path) -> bool:
|
|
27
|
-
try:
|
|
28
|
-
return left.resolve() == right.resolve()
|
|
29
|
-
except OSError:
|
|
30
|
-
return False
|
|
31
|
-
|
|
32
|
-
|
|
33
27
|
def convert_store(
|
|
34
28
|
file: Path,
|
|
35
29
|
output: Path,
|
|
@@ -56,7 +50,7 @@ def convert_store(
|
|
|
56
50
|
|
|
57
51
|
plan = converter.plan(file, output, src_format, dst_format, options)
|
|
58
52
|
for out in plan.outputs:
|
|
59
|
-
if
|
|
53
|
+
if same_path(file, out):
|
|
60
54
|
# Writing a Zarr store over its own source empties it before a
|
|
61
55
|
# single element has been read.
|
|
62
56
|
raise ValueError(f"Output path is the input: {out}")
|
|
@@ -60,7 +60,9 @@ def import_object(
|
|
|
60
60
|
console: Console,
|
|
61
61
|
) -> None:
|
|
62
62
|
target_file = _prepare_target_path(file, output_file, inplace, console)
|
|
63
|
-
|
|
63
|
+
suffixes = [x.lower() for x in input_file.suffixes]
|
|
64
|
+
# A gzipped matrix is still a matrix: `counts.mtx.gz` imports as `.mtx`.
|
|
65
|
+
ext = suffixes[-2] if len(suffixes) > 1 and suffixes[-1] == ".gz" else input_file.suffix.lower()
|
|
64
66
|
|
|
65
67
|
if ext not in EXTENSION_FORMAT:
|
|
66
68
|
raise ValueError(
|
|
@@ -2,13 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
Writes a whole new store rather than editing one in place, for the same
|
|
4
4
|
reason `subset` does: a conversion that fails half way leaves the original
|
|
5
|
-
untouched
|
|
6
|
-
|
|
5
|
+
untouched. The new store is written beside its destination and renamed into
|
|
6
|
+
place only once complete (see `adata.util.staging`), so `--inplace` replaces a
|
|
7
|
+
finished file rather than editing a live one, and the original is deleted only
|
|
8
|
+
after its successor is in place.
|
|
7
9
|
"""
|
|
8
10
|
|
|
9
11
|
from __future__ import annotations
|
|
10
12
|
|
|
11
|
-
import shutil
|
|
12
13
|
from pathlib import Path
|
|
13
14
|
from typing import Any, List, Optional
|
|
14
15
|
|
|
@@ -25,15 +26,16 @@ from adata.core.recast import (
|
|
|
25
26
|
from adata.elements import spec
|
|
26
27
|
from adata.elements.write import ensure_anndata_skeleton
|
|
27
28
|
from adata.storage import copy_tree, detect_backend, is_group, open_store
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
#: every 2-D array in the file: obsm/varm hold embeddings, whose dtype is
|
|
31
|
-
#: rarely what anyone is trying to shrink. Those still convert by path.
|
|
32
|
-
ALL_PREFIXES = ("X", "layers", "raw/X")
|
|
33
|
-
|
|
29
|
+
from adata.util.path import same_path
|
|
30
|
+
from adata.util.staging import staged_outputs
|
|
34
31
|
|
|
35
32
|
def discover_matrices(root: Any) -> List[str]:
|
|
36
|
-
"""Entry paths `--all` resolves to, in a stable order.
|
|
33
|
+
"""Entry paths `--all` resolves to, in a stable order.
|
|
34
|
+
|
|
35
|
+
The matrices an AnnData object is built from, not every 2-D array in the
|
|
36
|
+
file: obsm/varm hold embeddings, whose dtype is rarely what anyone is
|
|
37
|
+
trying to shrink. Those still convert by path.
|
|
38
|
+
"""
|
|
37
39
|
found: List[str] = []
|
|
38
40
|
if "X" in root:
|
|
39
41
|
found.append("X")
|
|
@@ -44,14 +46,6 @@ def discover_matrices(root: Any) -> List[str]:
|
|
|
44
46
|
return found
|
|
45
47
|
|
|
46
48
|
|
|
47
|
-
def _same_path(left: Path, right: Path) -> bool:
|
|
48
|
-
"""Do these name the same store, through symlinks and `..` alike?"""
|
|
49
|
-
try:
|
|
50
|
-
return left.resolve() == right.resolve()
|
|
51
|
-
except OSError: # pragma: no cover - unresolvable path
|
|
52
|
-
return left.absolute() == right.absolute()
|
|
53
|
-
|
|
54
|
-
|
|
55
49
|
def _resolve(root: Any, path: str) -> Any:
|
|
56
50
|
obj = root
|
|
57
51
|
for part in path.split("/"):
|
|
@@ -96,7 +90,7 @@ def recast_store(
|
|
|
96
90
|
else None
|
|
97
91
|
)
|
|
98
92
|
|
|
99
|
-
if output is not None and not inplace and
|
|
93
|
+
if output is not None and not inplace and same_path(file, output):
|
|
100
94
|
# Opening the destination "w" clears it while the source is still
|
|
101
95
|
# being read from it. HDF5 refuses; Zarr does not, and quietly
|
|
102
96
|
# produced an empty store where the data used to be.
|
|
@@ -105,69 +99,54 @@ def recast_store(
|
|
|
105
99
|
"it, which writes to a temporary file first."
|
|
106
100
|
)
|
|
107
101
|
|
|
108
|
-
if inplace
|
|
109
|
-
backend = detect_backend(file)
|
|
110
|
-
if backend == "zarr":
|
|
111
|
-
base = file.stem if file.suffix else file.name
|
|
112
|
-
dst_path = file.with_name(f"{base}.recast-tmp.zarr")
|
|
113
|
-
else:
|
|
114
|
-
dst_path = file.with_name(f"{file.name}.recast-tmp")
|
|
115
|
-
if dst_path.exists():
|
|
116
|
-
raise FileExistsError(f"Temporary path already exists: {dst_path}")
|
|
117
|
-
else:
|
|
118
|
-
dst_path = output
|
|
102
|
+
dst_path = file if inplace else Path(output)
|
|
119
103
|
|
|
120
104
|
if zarr_format is None and detect_backend(file) == "zarr":
|
|
121
105
|
with open_store(file, "r") as probe:
|
|
122
106
|
zarr_format = probe.zarr_format
|
|
123
107
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
108
|
+
# The source is opened inside the staging block, so it is closed before
|
|
109
|
+
# `--inplace` renames it aside.
|
|
110
|
+
with staged_outputs([dst_path]) as (temp,):
|
|
111
|
+
with open_store(file, "r") as src_store:
|
|
112
|
+
src = src_store.root
|
|
113
|
+
targets = discover_matrices(src) if convert_all else list(entries or [])
|
|
114
|
+
if not targets:
|
|
115
|
+
raise ValueError(
|
|
116
|
+
"No matrices selected. Name one (e.g. `X`) or pass --all."
|
|
117
|
+
)
|
|
118
|
+
# Every check first, before the destination exists. A refusal must
|
|
119
|
+
# not leave behind a store holding a copy of obs and var and nothing
|
|
120
|
+
# else -- the caller cannot tell that from a finished conversion.
|
|
121
|
+
plans = {
|
|
122
|
+
path: plan_conversion(
|
|
123
|
+
_resolve(src, path),
|
|
124
|
+
path,
|
|
125
|
+
dtype=data_dtype,
|
|
126
|
+
index_dtype=index_dtype,
|
|
127
|
+
layout=layout,
|
|
128
|
+
chunk=chunk,
|
|
129
|
+
force=force,
|
|
130
|
+
console=console,
|
|
131
|
+
)
|
|
132
|
+
for path in targets
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
console.print(
|
|
136
|
+
f"[cyan]Converting {len(targets)} matrix/matrices:[/] "
|
|
137
|
+
+ ", ".join(targets)
|
|
144
138
|
)
|
|
145
|
-
for path in targets
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
console.print(
|
|
149
|
-
f"[cyan]Converting {len(targets)} matrix/matrices:[/] "
|
|
150
|
-
+ ", ".join(targets)
|
|
151
|
-
)
|
|
152
139
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
140
|
+
with open_store(temp, "w", zarr_format=zarr_format) as dst_store:
|
|
141
|
+
dst = dst_store.root
|
|
142
|
+
_write(
|
|
143
|
+
src, dst, plans,
|
|
144
|
+
chunk=chunk, chunk_rows=chunk_rows, in_memory=in_memory,
|
|
145
|
+
console=console,
|
|
146
|
+
)
|
|
147
|
+
ensure_anndata_skeleton(dst)
|
|
161
148
|
|
|
162
149
|
if inplace:
|
|
163
|
-
if file.is_dir():
|
|
164
|
-
shutil.rmtree(file)
|
|
165
|
-
elif file.exists():
|
|
166
|
-
file.unlink()
|
|
167
|
-
if dst_path.is_dir():
|
|
168
|
-
shutil.move(str(dst_path), str(file))
|
|
169
|
-
else:
|
|
170
|
-
dst_path.replace(file)
|
|
171
150
|
console.print(f"[green]Converted[/] {file}")
|
|
172
151
|
else:
|
|
173
152
|
console.print(f"[green]Wrote[/] {dst_path}")
|
|
@@ -4,6 +4,11 @@ Requested in issue #2, modelled on cellgeni/scraft's `split_h5ad` but
|
|
|
4
4
|
streaming: the source is never loaded into memory, only the column being split
|
|
5
5
|
on is read, and each output is produced by the same subset machinery used by
|
|
6
6
|
`adata subset`, fanned out so every matrix is read once for all groups.
|
|
7
|
+
|
|
8
|
+
Every output is written beside its destination and renamed into place only
|
|
9
|
+
once all of them are complete, so a split that fails leaves no partial stores
|
|
10
|
+
(and existing outputs as they were). The manifest is written last, so it
|
|
11
|
+
exists only when everything it lists does.
|
|
7
12
|
"""
|
|
8
13
|
|
|
9
14
|
from __future__ import annotations
|
|
@@ -19,6 +24,8 @@ from adata.core.select import group_indices
|
|
|
19
24
|
from adata.core.subset import split_h5ad
|
|
20
25
|
from adata.storage import detect_backend, open_store
|
|
21
26
|
from adata.util.names import unique_names
|
|
27
|
+
from adata.util.path import same_path
|
|
28
|
+
from adata.util.staging import staged_outputs
|
|
22
29
|
|
|
23
30
|
|
|
24
31
|
def split_store(
|
|
@@ -78,6 +85,13 @@ def split_store(
|
|
|
78
85
|
for label, out_path, count in planned:
|
|
79
86
|
console.print(f" {label!r} -> {out_path} ({count} {axis})")
|
|
80
87
|
|
|
88
|
+
for _, out_path, _ in planned:
|
|
89
|
+
if same_path(file, out_path):
|
|
90
|
+
raise ValueError(
|
|
91
|
+
f"Output path is the input: {out_path}. Choose another "
|
|
92
|
+
"--output-dir."
|
|
93
|
+
)
|
|
94
|
+
|
|
81
95
|
if dry_run:
|
|
82
96
|
console.print("[yellow]Dry run: nothing written.[/]")
|
|
83
97
|
return planned
|
|
@@ -86,20 +100,21 @@ def split_store(
|
|
|
86
100
|
|
|
87
101
|
# One call for every group, so each matrix is read once rather than once
|
|
88
102
|
# per group.
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
+
with staged_outputs([out_path for _, out_path, _ in planned]) as temps:
|
|
104
|
+
split_h5ad(
|
|
105
|
+
file,
|
|
106
|
+
[
|
|
107
|
+
(
|
|
108
|
+
temp,
|
|
109
|
+
np.sort(groups[label]) if axis == "obs" else None,
|
|
110
|
+
np.sort(groups[label]) if axis == "var" else None,
|
|
111
|
+
)
|
|
112
|
+
for (label, _, _), temp in zip(planned, temps)
|
|
113
|
+
],
|
|
114
|
+
chunk_rows=chunk_rows,
|
|
115
|
+
console=console,
|
|
116
|
+
zarr_format=zarr_format,
|
|
117
|
+
)
|
|
103
118
|
|
|
104
119
|
if manifest:
|
|
105
120
|
_write_manifest(file, output_dir, column, planned, console)
|
|
@@ -17,7 +17,7 @@ from typing import Dict, FrozenSet, List, Optional
|
|
|
17
17
|
|
|
18
18
|
import h5py
|
|
19
19
|
|
|
20
|
-
from adata.conversion import tenx
|
|
20
|
+
from adata.conversion import mtx, tenx
|
|
21
21
|
from adata.conversion.anndata_copy import AnnDataCopy
|
|
22
22
|
from adata.conversion.base import ConversionPlan, Converter, Format, Options
|
|
23
23
|
from adata.storage import has_valid_anndata_root_attrs, is_zarr_path
|
|
@@ -74,6 +74,15 @@ FORMATS: Dict[str, Format] = {
|
|
|
74
74
|
importer=tenx.TenxImport(),
|
|
75
75
|
exporter=tenx.TenxExport(),
|
|
76
76
|
),
|
|
77
|
+
# A folder, so no extension: chosen as output with `--to 10x-mtx`.
|
|
78
|
+
"10x-mtx": Format(
|
|
79
|
+
name="10x-mtx",
|
|
80
|
+
description="10x Genomics / STARsolo Matrix Market folder",
|
|
81
|
+
extensions=(),
|
|
82
|
+
sniff=mtx.sniff,
|
|
83
|
+
importer=mtx.MtxImport(),
|
|
84
|
+
exporter=mtx.MtxExport(),
|
|
85
|
+
),
|
|
77
86
|
}
|
|
78
87
|
|
|
79
88
|
_ANNDATA_COPY = AnnDataCopy()
|
|
@@ -106,6 +115,12 @@ def detect_format(path: Path) -> str:
|
|
|
106
115
|
for fmt in FORMATS.values():
|
|
107
116
|
if (fmt.anndata or fmt.sniff is None) and suffix in fmt.extensions:
|
|
108
117
|
return fmt.name
|
|
118
|
+
if suffix in (".mtx", ".gz") and path.name.lower().endswith((".mtx", ".mtx.gz")):
|
|
119
|
+
raise ValueError(
|
|
120
|
+
f"'{path.name}' is a single Matrix Market file, which `convert` "
|
|
121
|
+
"does not read on its own: give the folder holding it with its "
|
|
122
|
+
"barcodes and features, or use `adata import sparse`."
|
|
123
|
+
)
|
|
109
124
|
raise ValueError(
|
|
110
125
|
f"Cannot tell what format '{path}' is. Pass --from, one of: {_names()}."
|
|
111
126
|
)
|
|
@@ -130,15 +145,21 @@ def output_format(path: Path, requested: Optional[str] = None) -> str:
|
|
|
130
145
|
if by_extension is None:
|
|
131
146
|
raise ValueError(
|
|
132
147
|
f"Cannot tell the output format from '{path.name}'. "
|
|
133
|
-
f"Use a known extension or pass --to, one of: {_names()}
|
|
148
|
+
f"Use a known extension or pass --to, one of: {_names()} "
|
|
149
|
+
"(--to 10x-mtx for a folder)."
|
|
134
150
|
)
|
|
135
151
|
return by_extension
|
|
136
152
|
|
|
137
153
|
if by_extension is not None and by_extension != requested:
|
|
154
|
+
extensions = FORMATS[requested].extensions
|
|
138
155
|
raise ValueError(
|
|
139
156
|
f"--to {requested} contradicts the output's extension, which means "
|
|
140
|
-
f"{by_extension}. Name the output
|
|
141
|
-
|
|
157
|
+
f"{by_extension}. Name the output "
|
|
158
|
+
+ (
|
|
159
|
+
f"with one of: {', '.join(extensions)}."
|
|
160
|
+
if extensions
|
|
161
|
+
else "as a folder, without an extension."
|
|
162
|
+
)
|
|
142
163
|
)
|
|
143
164
|
if (requested == "zarr") != (suffix == ".zarr"):
|
|
144
165
|
raise ValueError(
|