pyadata-cli 0.6.1__tar.gz → 0.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/PKG-INFO +17 -7
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/README.md +16 -6
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/pyproject.toml +1 -1
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/cli.py +315 -24
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/commands/__init__.py +1 -0
- pyadata_cli-0.7.1/src/adata/commands/convert.py +71 -0
- pyadata_cli-0.6.1/src/adata/commands/convert.py → pyadata_cli-0.7.1/src/adata/commands/recast.py +55 -76
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/commands/split.py +30 -43
- pyadata_cli-0.7.1/src/adata/conversion/__init__.py +186 -0
- pyadata_cli-0.7.1/src/adata/conversion/anndata_copy.py +90 -0
- pyadata_cli-0.7.1/src/adata/conversion/base.py +98 -0
- pyadata_cli-0.7.1/src/adata/conversion/tenx.py +812 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/core/concat.py +23 -3
- pyadata_cli-0.6.1/src/adata/core/convert.py → pyadata_cli-0.7.1/src/adata/core/recast.py +71 -16
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/core/subset.py +58 -68
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/storage/__init__.py +190 -12
- pyadata_cli-0.7.1/src/adata/util/names.py +69 -0
- pyadata_cli-0.7.1/src/adata/util/path.py +19 -0
- pyadata_cli-0.7.1/src/adata/util/staging.py +99 -0
- pyadata_cli-0.6.1/src/adata/util/path.py +0 -9
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/LICENSE +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/__init__.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/commands/concat.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/commands/create.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/commands/export.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/commands/import_data.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/commands/info.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/commands/ls.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/commands/subset.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/core/__init__.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/core/info.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/core/query.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/core/read.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/core/select.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/elements/__init__.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/elements/read.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/elements/spec.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/elements/strings.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/elements/write.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/formats/__init__.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/formats/array.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/formats/common.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/formats/dataframe.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/formats/image.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/formats/json_data.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/formats/sparse.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/formats/validate.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/info.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/py.typed +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/read.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.1}/src/adata/util/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pyadata-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.1
|
|
4
4
|
Summary: Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores
|
|
5
5
|
Keywords: anndata,h5ad,zarr,bioinformatics,single-cell,cli
|
|
6
6
|
Author: Aljes Binkevich
|
|
@@ -48,7 +48,7 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with
|
|
|
48
48
|
- Auto-detects `.h5ad` files vs `.zarr` directories
|
|
49
49
|
- Chunked processing for dense and sparse matrices (CSR/CSC)
|
|
50
50
|
- Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
|
|
51
|
-
- Converts between HDF5 and Zarr (v2 and v3) in either direction
|
|
51
|
+
- Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices
|
|
52
52
|
- Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
|
|
53
53
|
|
|
54
54
|
**Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
|
|
@@ -96,7 +96,8 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
|
|
|
96
96
|
- `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
|
|
97
97
|
- `split` – write one store per distinct value of an annotation column, with a CSV manifest.
|
|
98
98
|
- `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
|
|
99
|
-
- `convert` – change a
|
|
99
|
+
- `convert` – change a store's format: h5ad ↔ Zarr v2/v3, and 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) in and out, streaming, with the output written beside its destination and moved into place only once complete.
|
|
100
|
+
- `recast` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced. Called `convert` until 0.7.0.
|
|
100
101
|
- `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
|
|
101
102
|
- `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
|
|
102
103
|
|
|
@@ -122,16 +123,25 @@ adata concat per_sample/*.h5ad -o merged.h5ad --join outer --label sample
|
|
|
122
123
|
### Shrinking a store, and making encodings agree
|
|
123
124
|
|
|
124
125
|
```bash
|
|
125
|
-
adata
|
|
126
|
-
adata
|
|
127
|
-
adata
|
|
126
|
+
adata recast data.h5ad X --inplace --dtype float32
|
|
127
|
+
adata recast data.h5ad --all -o small.h5ad --dtype float32 --indices-dtype int32
|
|
128
|
+
adata recast data.h5ad X -o csc.h5ad --layout csc
|
|
128
129
|
```
|
|
129
130
|
|
|
130
|
-
Counts written as float64 halve with no loss — and `
|
|
131
|
+
Counts written as float64 halve with no loss — and `recast` proves that
|
|
131
132
|
before it writes, by casting every value and casting it back. `concat`
|
|
132
133
|
refuses inputs whose matrices disagree about CSR versus CSC; `--layout` is
|
|
133
134
|
how you make them agree.
|
|
134
135
|
|
|
136
|
+
### Changing format
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
adata convert data.h5ad -o data.zarr
|
|
140
|
+
adata convert data.zarr -o data.h5ad --compression gzip
|
|
141
|
+
adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
|
|
142
|
+
adata convert counts.h5ad -o for_cellbender.h5 --layer counts
|
|
143
|
+
```
|
|
144
|
+
|
|
135
145
|
## Documentation
|
|
136
146
|
|
|
137
147
|
- [Get started](docs/GET_STARTED.md) — a short tutorial
|
|
@@ -8,7 +8,7 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with
|
|
|
8
8
|
- Auto-detects `.h5ad` files vs `.zarr` directories
|
|
9
9
|
- Chunked processing for dense and sparse matrices (CSR/CSC)
|
|
10
10
|
- Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
|
|
11
|
-
- Converts between HDF5 and Zarr (v2 and v3) in either direction
|
|
11
|
+
- Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices
|
|
12
12
|
- Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
|
|
13
13
|
|
|
14
14
|
**Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
|
|
@@ -56,7 +56,8 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
|
|
|
56
56
|
- `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
|
|
57
57
|
- `split` – write one store per distinct value of an annotation column, with a CSV manifest.
|
|
58
58
|
- `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
|
|
59
|
-
- `convert` – change a
|
|
59
|
+
- `convert` – change a store's format: h5ad ↔ Zarr v2/v3, and 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) in and out, streaming, with the output written beside its destination and moved into place only once complete.
|
|
60
|
+
- `recast` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced. Called `convert` until 0.7.0.
|
|
60
61
|
- `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
|
|
61
62
|
- `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
|
|
62
63
|
|
|
@@ -82,16 +83,25 @@ adata concat per_sample/*.h5ad -o merged.h5ad --join outer --label sample
|
|
|
82
83
|
### Shrinking a store, and making encodings agree
|
|
83
84
|
|
|
84
85
|
```bash
|
|
85
|
-
adata
|
|
86
|
-
adata
|
|
87
|
-
adata
|
|
86
|
+
adata recast data.h5ad X --inplace --dtype float32
|
|
87
|
+
adata recast data.h5ad --all -o small.h5ad --dtype float32 --indices-dtype int32
|
|
88
|
+
adata recast data.h5ad X -o csc.h5ad --layout csc
|
|
88
89
|
```
|
|
89
90
|
|
|
90
|
-
Counts written as float64 halve with no loss — and `
|
|
91
|
+
Counts written as float64 halve with no loss — and `recast` proves that
|
|
91
92
|
before it writes, by casting every value and casting it back. `concat`
|
|
92
93
|
refuses inputs whose matrices disagree about CSR versus CSC; `--layout` is
|
|
93
94
|
how you make them agree.
|
|
94
95
|
|
|
96
|
+
### Changing format
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
adata convert data.h5ad -o data.zarr
|
|
100
|
+
adata convert data.zarr -o data.h5ad --compression gzip
|
|
101
|
+
adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
|
|
102
|
+
adata convert counts.h5ad -o for_cellbender.h5 --layer counts
|
|
103
|
+
```
|
|
104
|
+
|
|
95
105
|
## Documentation
|
|
96
106
|
|
|
97
107
|
- [Get started](docs/GET_STARTED.md) — a short tutorial
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# so the distribution is published as `pyadata-cli`. The import package
|
|
4
4
|
# and the command are both still `adata`.
|
|
5
5
|
name = "pyadata-cli"
|
|
6
|
-
version = "0.
|
|
6
|
+
version = "0.7.1"
|
|
7
7
|
description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores"
|
|
8
8
|
readme = "README.md"
|
|
9
9
|
requires-python = ">=3.12"
|
|
@@ -6,11 +6,13 @@ from typing import Optional, Sequence, List
|
|
|
6
6
|
from rich.console import Console
|
|
7
7
|
import typer
|
|
8
8
|
|
|
9
|
-
from adata.
|
|
9
|
+
from adata.conversion import FORMATS, Options
|
|
10
|
+
from adata.core.recast import LAYOUTS
|
|
10
11
|
from adata.commands import (
|
|
11
12
|
MERGE_CHOICES,
|
|
12
13
|
concat_stores,
|
|
13
14
|
convert_store,
|
|
15
|
+
recast_store,
|
|
14
16
|
create_store,
|
|
15
17
|
split_store,
|
|
16
18
|
list_store,
|
|
@@ -26,7 +28,7 @@ from adata.commands import export_image as export_image_cmd
|
|
|
26
28
|
|
|
27
29
|
app = typer.Typer(
|
|
28
30
|
help="Streaming CLI for huge AnnData .h5ad and .zarr stores "
|
|
29
|
-
"(view, ls, create, subset, split, concat, convert, export, import)."
|
|
31
|
+
"(view, ls, create, subset, split, concat, convert, recast, export, import)."
|
|
30
32
|
)
|
|
31
33
|
# Use stderr for status/progress to keep stdout clean for data output
|
|
32
34
|
# force_terminal=True ensures Rich output is visible even in non-TTY environments
|
|
@@ -345,6 +347,10 @@ def subset(
|
|
|
345
347
|
adata subset data.h5ad -o out.h5ad -q "n_counts > 1000 and cluster in A,B"
|
|
346
348
|
adata subset data.h5ad -o out.h5ad --var-query "highly_variable == True"
|
|
347
349
|
"""
|
|
350
|
+
if zarr_format is not None and zarr_format not in (2, 3):
|
|
351
|
+
console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.")
|
|
352
|
+
raise typer.Exit(code=1)
|
|
353
|
+
|
|
348
354
|
if obs is None and var is None and obs_query is None and var_query is None:
|
|
349
355
|
console.print(
|
|
350
356
|
"[bold red]Error:[/] Provide at least one of --obs, --var, "
|
|
@@ -431,7 +437,9 @@ def concat(
|
|
|
431
437
|
1024, "--chunk", "-C", help="Row chunk size while streaming"
|
|
432
438
|
),
|
|
433
439
|
zarr_format: Optional[int] = typer.Option(
|
|
434
|
-
None,
|
|
440
|
+
None,
|
|
441
|
+
"--zarr-format",
|
|
442
|
+
help="Zarr spec version to write (defaults to the inputs' when they agree)",
|
|
435
443
|
),
|
|
436
444
|
) -> None:
|
|
437
445
|
"""
|
|
@@ -445,6 +453,10 @@ def concat(
|
|
|
445
453
|
adata concat *.h5ad -o merged.h5ad --join outer --label sample
|
|
446
454
|
adata concat a.h5ad b.h5ad -o m.h5ad --keys a,b --index-unique - --uns-merge same
|
|
447
455
|
"""
|
|
456
|
+
if zarr_format is not None and zarr_format not in (2, 3):
|
|
457
|
+
console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.")
|
|
458
|
+
raise typer.Exit(code=1)
|
|
459
|
+
|
|
448
460
|
for name, value in (("--merge", merge), ("--uns-merge", uns_merge)):
|
|
449
461
|
if value is not None and value not in MERGE_CHOICES:
|
|
450
462
|
console.print(
|
|
@@ -481,8 +493,285 @@ def concat(
|
|
|
481
493
|
# ============================================================================
|
|
482
494
|
# CONVERT command
|
|
483
495
|
# ============================================================================
|
|
484
|
-
|
|
496
|
+
#: The flags `recast` has and format conversion does not. Until 1.0.0, any of
|
|
497
|
+
#: them on `convert` means the old meaning of the command, and it forwards.
|
|
498
|
+
_RECAST_FLAGS = {
|
|
499
|
+
"inplace": "--inplace",
|
|
500
|
+
"convert_all": "--all",
|
|
501
|
+
"dtype": "--dtype",
|
|
502
|
+
"indices_dtype": "--indices-dtype",
|
|
503
|
+
"layout": "--layout",
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
#: Flags that only shape a recast; given without one of the above they have
|
|
507
|
+
#: nothing to act on.
|
|
508
|
+
_RECAST_ONLY_FLAGS = {"in_memory": "--in-memory", "chunk_rows": "--chunk"}
|
|
509
|
+
|
|
510
|
+
#: Format-conversion flags, by parameter name, as `Options.given` spells them.
|
|
511
|
+
_CONVERT_FLAGS = {
|
|
512
|
+
"zarr_format": "--zarr-format",
|
|
513
|
+
"compression": "--compression",
|
|
514
|
+
"force": "--force",
|
|
515
|
+
"overwrite": "--overwrite",
|
|
516
|
+
"genome": "--genome",
|
|
517
|
+
"var_names": "--var-names",
|
|
518
|
+
"make_unique": "--make-unique",
|
|
519
|
+
"feature_type": "--feature-type",
|
|
520
|
+
"split_feature_types": "--split-feature-types",
|
|
521
|
+
"layer": "--layer",
|
|
522
|
+
"gene_ids_column": "--gene-ids-column",
|
|
523
|
+
"gene_names_column": "--gene-names-column",
|
|
524
|
+
"feature_types_column": "--feature-types-column",
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
#: Flags that only make sense for format conversion; with recast's flags,
|
|
528
|
+
#: they mean two commands' worth of request at once.
|
|
529
|
+
_FORMAT_ONLY_FLAGS = {
|
|
530
|
+
"from_format": "--from",
|
|
531
|
+
"to_format": "--to",
|
|
532
|
+
**{
|
|
533
|
+
name: flag
|
|
534
|
+
for name, flag in _CONVERT_FLAGS.items()
|
|
535
|
+
if flag not in ("--force", "--zarr-format")
|
|
536
|
+
},
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def _given(ctx: typer.Context, flags: dict) -> List[str]:
|
|
541
|
+
"""The flags among `flags` that were actually typed, not defaulted."""
|
|
542
|
+
from click.core import ParameterSource
|
|
543
|
+
|
|
544
|
+
return [
|
|
545
|
+
flag
|
|
546
|
+
for name, flag in flags.items()
|
|
547
|
+
if ctx.get_parameter_source(name) not in (None, ParameterSource.DEFAULT)
|
|
548
|
+
]
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
@app.command("convert", context_settings={"allow_extra_args": True})
|
|
485
552
|
def convert(
|
|
553
|
+
ctx: typer.Context,
|
|
554
|
+
file: Path = typer.Argument(
|
|
555
|
+
...,
|
|
556
|
+
help="Input file or store",
|
|
557
|
+
exists=True,
|
|
558
|
+
readable=True,
|
|
559
|
+
dir_okay=True,
|
|
560
|
+
file_okay=True,
|
|
561
|
+
),
|
|
562
|
+
output: Optional[Path] = typer.Option(
|
|
563
|
+
None,
|
|
564
|
+
"--output",
|
|
565
|
+
"-o",
|
|
566
|
+
help="Output path; its extension picks the format (.h5ad, .zarr, .h5)",
|
|
567
|
+
dir_okay=True,
|
|
568
|
+
file_okay=True,
|
|
569
|
+
),
|
|
570
|
+
from_format: Optional[str] = typer.Option(
|
|
571
|
+
None,
|
|
572
|
+
"--from",
|
|
573
|
+
help=f"Input format, if it cannot be detected: {', '.join(FORMATS)}",
|
|
574
|
+
),
|
|
575
|
+
to_format: Optional[str] = typer.Option(
|
|
576
|
+
None,
|
|
577
|
+
"--to",
|
|
578
|
+
help=f"Output format, if the extension does not say: {', '.join(FORMATS)}",
|
|
579
|
+
),
|
|
580
|
+
zarr_format: Optional[int] = typer.Option(
|
|
581
|
+
None,
|
|
582
|
+
"--zarr-format",
|
|
583
|
+
help="Zarr spec version to write (defaults to the source store's, else 3)",
|
|
584
|
+
),
|
|
585
|
+
compression: Optional[str] = typer.Option(
|
|
586
|
+
None,
|
|
587
|
+
"--compression",
|
|
588
|
+
help="none, gzip or lzf for h5ad; none, gzip, zstd or blosc for zarr "
|
|
589
|
+
"(defaults to the source's)",
|
|
590
|
+
),
|
|
591
|
+
force: bool = typer.Option(
|
|
592
|
+
False,
|
|
593
|
+
"--force",
|
|
594
|
+
help="Convert despite a lossy or inflating result",
|
|
595
|
+
),
|
|
596
|
+
overwrite: bool = typer.Option(
|
|
597
|
+
False,
|
|
598
|
+
"--overwrite",
|
|
599
|
+
help="Replace an existing output",
|
|
600
|
+
),
|
|
601
|
+
genome: Optional[str] = typer.Option(
|
|
602
|
+
None,
|
|
603
|
+
"--genome",
|
|
604
|
+
help="10x input: the genome to keep (picks the group in a Cell Ranger 2 file)",
|
|
605
|
+
),
|
|
606
|
+
var_names: str = typer.Option(
|
|
607
|
+
"gene_symbols",
|
|
608
|
+
"--var-names",
|
|
609
|
+
help="10x input: name features by gene_symbols or gene_ids",
|
|
610
|
+
),
|
|
611
|
+
make_unique: bool = typer.Option(
|
|
612
|
+
False,
|
|
613
|
+
"--make-unique",
|
|
614
|
+
help="10x input: suffix repeated var names, as var_names_make_unique does",
|
|
615
|
+
),
|
|
616
|
+
feature_type: Optional[str] = typer.Option(
|
|
617
|
+
None,
|
|
618
|
+
"--feature-type",
|
|
619
|
+
help="10x input: keep only this feature type, e.g. 'Gene Expression'",
|
|
620
|
+
),
|
|
621
|
+
split_feature_types: bool = typer.Option(
|
|
622
|
+
False,
|
|
623
|
+
"--split-feature-types",
|
|
624
|
+
help="10x input: write one output per feature type, OUT.<type>.h5ad",
|
|
625
|
+
),
|
|
626
|
+
layer: Optional[str] = typer.Option(
|
|
627
|
+
None,
|
|
628
|
+
"--layer",
|
|
629
|
+
help="10x output: the matrix to write: X (default), raw/X, or a layer",
|
|
630
|
+
),
|
|
631
|
+
gene_ids_column: Optional[str] = typer.Option(
|
|
632
|
+
None,
|
|
633
|
+
"--gene-ids-column",
|
|
634
|
+
help="10x output: var column holding gene ids (default gene_ids)",
|
|
635
|
+
),
|
|
636
|
+
gene_names_column: Optional[str] = typer.Option(
|
|
637
|
+
None,
|
|
638
|
+
"--gene-names-column",
|
|
639
|
+
help="10x output: var column holding gene names (default: var_names)",
|
|
640
|
+
),
|
|
641
|
+
feature_types_column: Optional[str] = typer.Option(
|
|
642
|
+
None,
|
|
643
|
+
"--feature-types-column",
|
|
644
|
+
help="10x output: var column holding feature types (default feature_types)",
|
|
645
|
+
),
|
|
646
|
+
# Deprecated: recast's flags, so `convert` keeps working as it did until
|
|
647
|
+
# 1.0.0. Hidden from --help; the old matrix paths arrive in ctx.args.
|
|
648
|
+
inplace: bool = typer.Option(False, "--inplace", hidden=True),
|
|
649
|
+
convert_all: bool = typer.Option(False, "--all", hidden=True),
|
|
650
|
+
dtype: Optional[str] = typer.Option(None, "--dtype", hidden=True),
|
|
651
|
+
indices_dtype: Optional[str] = typer.Option(None, "--indices-dtype", hidden=True),
|
|
652
|
+
layout: Optional[str] = typer.Option(None, "--layout", hidden=True),
|
|
653
|
+
in_memory: bool = typer.Option(False, "--in-memory", hidden=True),
|
|
654
|
+
chunk_rows: int = typer.Option(1024, "--chunk", "-C", hidden=True),
|
|
655
|
+
) -> None:
|
|
656
|
+
"""
|
|
657
|
+
Convert a store to another format.
|
|
658
|
+
|
|
659
|
+
The output's extension picks the format: .h5ad, .zarr, or .h5 for a 10x
|
|
660
|
+
Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5). Between h5ad and
|
|
661
|
+
zarr every element is copied, streamed, keeping the source's chunking and
|
|
662
|
+
compression where the target can express them. A 10x matrix is read as
|
|
663
|
+
scanpy.read_10x_h5(gex_only=False) reads it, keeping integer counts, and
|
|
664
|
+
written as Cell Ranger writes it.
|
|
665
|
+
|
|
666
|
+
Nothing is written until every check has passed, and the output is moved
|
|
667
|
+
into place only once complete: a failed conversion leaves nothing
|
|
668
|
+
behind. An existing output is refused unless --overwrite is given.
|
|
669
|
+
|
|
670
|
+
To change a matrix's dtype, layout or density, use `recast`. Until 1.0.0,
|
|
671
|
+
`convert` given recast's flags still forwards to it, with a warning.
|
|
672
|
+
|
|
673
|
+
Examples:
|
|
674
|
+
adata convert data.h5ad -o data.zarr
|
|
675
|
+
adata convert data.zarr -o data.h5ad --compression gzip
|
|
676
|
+
adata convert data.zarr -o v3.zarr --zarr-format 3
|
|
677
|
+
adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad
|
|
678
|
+
adata convert raw_feature_bc_matrix.h5 -o rna.h5ad --feature-type "Gene Expression"
|
|
679
|
+
adata convert counts.h5ad -o for_cellbender.h5 --layer counts
|
|
680
|
+
"""
|
|
681
|
+
entries = list(ctx.args)
|
|
682
|
+
recast_flags = _given(ctx, _RECAST_FLAGS)
|
|
683
|
+
|
|
684
|
+
try:
|
|
685
|
+
if recast_flags:
|
|
686
|
+
mixed = _given(ctx, _FORMAT_ONLY_FLAGS)
|
|
687
|
+
if mixed:
|
|
688
|
+
raise ValueError(
|
|
689
|
+
f"{', '.join(mixed)} cannot be combined with "
|
|
690
|
+
f"{', '.join(recast_flags)}: the first change a store's "
|
|
691
|
+
"format, the second its matrices. Run `adata recast` for "
|
|
692
|
+
"the matrices."
|
|
693
|
+
)
|
|
694
|
+
console.print(
|
|
695
|
+
"[yellow]Warning:[/] 'convert' with "
|
|
696
|
+
f"{', '.join(recast_flags)} is deprecated and will be removed "
|
|
697
|
+
"in 1.0.0; use 'recast' instead.",
|
|
698
|
+
)
|
|
699
|
+
if not inplace and output is None:
|
|
700
|
+
raise ValueError("Output file is required. Use --output/-o or --inplace.")
|
|
701
|
+
if layout is not None and layout not in LAYOUTS:
|
|
702
|
+
raise ValueError(f"--layout must be one of: {', '.join(LAYOUTS)}.")
|
|
703
|
+
if zarr_format is not None and zarr_format not in (2, 3):
|
|
704
|
+
raise ValueError("--zarr-format must be 2 or 3.")
|
|
705
|
+
recast_store(
|
|
706
|
+
file=file,
|
|
707
|
+
entries=entries or None,
|
|
708
|
+
output=output,
|
|
709
|
+
console=console,
|
|
710
|
+
dtype=dtype,
|
|
711
|
+
indices_dtype=indices_dtype,
|
|
712
|
+
layout=layout,
|
|
713
|
+
convert_all=convert_all,
|
|
714
|
+
inplace=inplace,
|
|
715
|
+
chunk_rows=chunk_rows,
|
|
716
|
+
in_memory=in_memory,
|
|
717
|
+
force=force,
|
|
718
|
+
zarr_format=zarr_format,
|
|
719
|
+
)
|
|
720
|
+
return
|
|
721
|
+
|
|
722
|
+
if entries:
|
|
723
|
+
hint = (
|
|
724
|
+
f" To write it, pass -o {entries[0]}."
|
|
725
|
+
if output is None and len(entries) == 1
|
|
726
|
+
else ""
|
|
727
|
+
)
|
|
728
|
+
raise ValueError(
|
|
729
|
+
f"Unexpected argument{'s' if len(entries) > 1 else ''}: "
|
|
730
|
+
f"{' '.join(entries)}. convert takes one input.{hint} To change "
|
|
731
|
+
"a matrix's dtype or layout, use `adata recast`."
|
|
732
|
+
)
|
|
733
|
+
stray = _given(ctx, _RECAST_ONLY_FLAGS)
|
|
734
|
+
if stray:
|
|
735
|
+
raise ValueError(
|
|
736
|
+
f"{', '.join(stray)} {'applies' if len(stray) == 1 else 'apply'} "
|
|
737
|
+
"to `adata recast`, not to format conversion."
|
|
738
|
+
)
|
|
739
|
+
if output is None:
|
|
740
|
+
raise ValueError("Output path is required: pass -o/--output.")
|
|
741
|
+
|
|
742
|
+
convert_store(
|
|
743
|
+
file,
|
|
744
|
+
output,
|
|
745
|
+
console,
|
|
746
|
+
src_format=from_format,
|
|
747
|
+
dst_format=to_format,
|
|
748
|
+
options=Options(
|
|
749
|
+
given=frozenset(_given(ctx, _CONVERT_FLAGS)),
|
|
750
|
+
zarr_format=zarr_format,
|
|
751
|
+
compression=compression,
|
|
752
|
+
force=force,
|
|
753
|
+
overwrite=overwrite,
|
|
754
|
+
genome=genome,
|
|
755
|
+
var_names=var_names,
|
|
756
|
+
make_unique=make_unique,
|
|
757
|
+
feature_type=feature_type,
|
|
758
|
+
split_feature_types=split_feature_types,
|
|
759
|
+
layer=layer,
|
|
760
|
+
gene_ids_column=gene_ids_column,
|
|
761
|
+
gene_names_column=gene_names_column,
|
|
762
|
+
feature_types_column=feature_types_column,
|
|
763
|
+
),
|
|
764
|
+
)
|
|
765
|
+
except Exception as e:
|
|
766
|
+
console.print(f"[bold red]Error:[/] {e}")
|
|
767
|
+
raise typer.Exit(code=1)
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
# ============================================================================
|
|
771
|
+
# RECAST command
|
|
772
|
+
# ============================================================================
|
|
773
|
+
@app.command("recast")
|
|
774
|
+
def recast(
|
|
486
775
|
file: Path = typer.Argument(
|
|
487
776
|
...,
|
|
488
777
|
help="Input .h5ad/.zarr",
|
|
@@ -493,7 +782,7 @@ def convert(
|
|
|
493
782
|
),
|
|
494
783
|
entries: Optional[List[str]] = typer.Argument(
|
|
495
784
|
None,
|
|
496
|
-
help="Matrix paths to
|
|
785
|
+
help="Matrix paths to recast, e.g. 'X', 'layers/counts', 'raw/X'",
|
|
497
786
|
),
|
|
498
787
|
output: Optional[Path] = typer.Option(
|
|
499
788
|
None,
|
|
@@ -511,7 +800,7 @@ def convert(
|
|
|
511
800
|
convert_all: bool = typer.Option(
|
|
512
801
|
False,
|
|
513
802
|
"--all",
|
|
514
|
-
help="
|
|
803
|
+
help="Recast X, every layer, and raw/X",
|
|
515
804
|
),
|
|
516
805
|
dtype: Optional[str] = typer.Option(
|
|
517
806
|
None,
|
|
@@ -531,7 +820,7 @@ def convert(
|
|
|
531
820
|
force: bool = typer.Option(
|
|
532
821
|
False,
|
|
533
822
|
"--force",
|
|
534
|
-
help="
|
|
823
|
+
help="Recast despite a lossy cast or a large size increase",
|
|
535
824
|
),
|
|
536
825
|
in_memory: bool = typer.Option(
|
|
537
826
|
False,
|
|
@@ -562,11 +851,13 @@ def convert(
|
|
|
562
851
|
both. Transposing streams by default, so it works on matrices too large
|
|
563
852
|
to load.
|
|
564
853
|
|
|
854
|
+
This was `convert` until 0.7.0; `convert` now changes a store's format.
|
|
855
|
+
|
|
565
856
|
Examples:
|
|
566
|
-
adata
|
|
567
|
-
adata
|
|
568
|
-
adata
|
|
569
|
-
adata
|
|
857
|
+
adata recast data.h5ad X -o out.h5ad --dtype float32
|
|
858
|
+
adata recast data.h5ad X -o out.h5ad --layout csc
|
|
859
|
+
adata recast data.h5ad X --inplace --dtype float32 --indices-dtype int32
|
|
860
|
+
adata recast data.h5ad --all -o out.h5ad --dtype float32
|
|
570
861
|
"""
|
|
571
862
|
if not inplace and output is None:
|
|
572
863
|
console.print(
|
|
@@ -586,7 +877,7 @@ def convert(
|
|
|
586
877
|
raise typer.Exit(code=1)
|
|
587
878
|
|
|
588
879
|
try:
|
|
589
|
-
|
|
880
|
+
recast_store(
|
|
590
881
|
file=file,
|
|
591
882
|
entries=list(entries) if entries else None,
|
|
592
883
|
output=output,
|
|
@@ -848,9 +1139,9 @@ def export_sparse(
|
|
|
848
1139
|
Export a sparse matrix (CSR/CSC) to Matrix Market (.mtx) format.
|
|
849
1140
|
|
|
850
1141
|
Examples:
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
1142
|
+
adata export sparse data.h5ad X -o matrix.mtx
|
|
1143
|
+
adata export sparse data.h5ad layers/counts -o counts.mtx
|
|
1144
|
+
adata export sparse data.h5ad X --head 100
|
|
854
1145
|
"""
|
|
855
1146
|
|
|
856
1147
|
try:
|
|
@@ -898,8 +1189,8 @@ def export_dict(
|
|
|
898
1189
|
Export a dict/group or scalar to JSON format.
|
|
899
1190
|
|
|
900
1191
|
Examples:
|
|
901
|
-
|
|
902
|
-
|
|
1192
|
+
adata export dict data.h5ad uns metadata.json
|
|
1193
|
+
adata export dict data.h5ad uns/colors colors.json
|
|
903
1194
|
"""
|
|
904
1195
|
|
|
905
1196
|
try:
|
|
@@ -938,7 +1229,7 @@ def export_image(
|
|
|
938
1229
|
The array should be 2D (H,W) or 3D (H,W,C) with C in {1,3,4}.
|
|
939
1230
|
|
|
940
1231
|
Examples:
|
|
941
|
-
|
|
1232
|
+
adata export image data.h5ad uns/spatial/image -o tissue.png
|
|
942
1233
|
"""
|
|
943
1234
|
|
|
944
1235
|
try:
|
|
@@ -1082,8 +1373,8 @@ def import_array(
|
|
|
1082
1373
|
Dimensions are validated against existing obs/var.
|
|
1083
1374
|
|
|
1084
1375
|
Examples:
|
|
1085
|
-
|
|
1086
|
-
|
|
1376
|
+
adata import array data.h5ad obsm/X_pca pca.npy -o output.h5ad
|
|
1377
|
+
adata import array data.h5ad X matrix.npy --inplace
|
|
1087
1378
|
"""
|
|
1088
1379
|
from adata.commands.import_data import _import_npy
|
|
1089
1380
|
|
|
@@ -1138,8 +1429,8 @@ def import_sparse(
|
|
|
1138
1429
|
Dimensions are validated against existing obs/var.
|
|
1139
1430
|
|
|
1140
1431
|
Examples:
|
|
1141
|
-
|
|
1142
|
-
|
|
1432
|
+
adata import sparse data.h5ad X matrix.mtx -o output.h5ad
|
|
1433
|
+
adata import sparse data.h5ad layers/counts counts.mtx --inplace
|
|
1143
1434
|
"""
|
|
1144
1435
|
from adata.commands.import_data import _import_mtx
|
|
1145
1436
|
|
|
@@ -1192,8 +1483,8 @@ def import_dict(
|
|
|
1192
1483
|
Import a JSON file into uns or other dict-like groups.
|
|
1193
1484
|
|
|
1194
1485
|
Examples:
|
|
1195
|
-
|
|
1196
|
-
|
|
1486
|
+
adata import dict data.h5ad uns/metadata config.json -o output.h5ad
|
|
1487
|
+
adata import dict data.h5ad uns settings.json --inplace
|
|
1197
1488
|
"""
|
|
1198
1489
|
from adata.commands.import_data import _import_json
|
|
1199
1490
|
|
|
@@ -6,4 +6,5 @@ from adata.commands.ls import list_store
|
|
|
6
6
|
from adata.commands.create import create_store
|
|
7
7
|
from adata.commands.split import split_store
|
|
8
8
|
from adata.commands.concat import MERGE_CHOICES, MERGE_STRATEGIES, concat_stores
|
|
9
|
+
from adata.commands.recast import recast_store
|
|
9
10
|
from adata.commands.convert import convert_store
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""The `convert` command: rewrite a store in another format.
|
|
2
|
+
|
|
3
|
+
Every refusal happens while building the plan, before an output exists, and
|
|
4
|
+
the outputs are written beside their destination and moved into place only
|
|
5
|
+
once complete. A conversion that fails therefore leaves nothing behind, and
|
|
6
|
+
never a half-written store that the next command would read as a whole one.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import List, Optional
|
|
13
|
+
|
|
14
|
+
from rich.console import Console
|
|
15
|
+
|
|
16
|
+
from adata.conversion import (
|
|
17
|
+
Options,
|
|
18
|
+
check_options,
|
|
19
|
+
converter_for,
|
|
20
|
+
detect_format,
|
|
21
|
+
output_format,
|
|
22
|
+
)
|
|
23
|
+
from adata.util.path import same_path
|
|
24
|
+
from adata.util.staging import staged_outputs
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def convert_store(
|
|
28
|
+
file: Path,
|
|
29
|
+
output: Path,
|
|
30
|
+
console: Console,
|
|
31
|
+
*,
|
|
32
|
+
src_format: Optional[str] = None,
|
|
33
|
+
dst_format: Optional[str] = None,
|
|
34
|
+
options: Options = Options(),
|
|
35
|
+
) -> List[Path]:
|
|
36
|
+
"""Convert `file` into `output`; return the paths written."""
|
|
37
|
+
file, output = Path(file), Path(output)
|
|
38
|
+
if str(output) == "-":
|
|
39
|
+
raise ValueError(
|
|
40
|
+
"convert writes a store, which cannot go to stdout; give a path "
|
|
41
|
+
"with -o."
|
|
42
|
+
)
|
|
43
|
+
if options.zarr_format not in (None, 2, 3):
|
|
44
|
+
raise ValueError("--zarr-format must be 2 or 3.")
|
|
45
|
+
|
|
46
|
+
src_format = src_format or detect_format(file)
|
|
47
|
+
dst_format = output_format(output, dst_format)
|
|
48
|
+
converter = converter_for(src_format, dst_format)
|
|
49
|
+
check_options(converter, options, src_format, dst_format)
|
|
50
|
+
|
|
51
|
+
plan = converter.plan(file, output, src_format, dst_format, options)
|
|
52
|
+
for out in plan.outputs:
|
|
53
|
+
if same_path(file, out):
|
|
54
|
+
# Writing a Zarr store over its own source empties it before a
|
|
55
|
+
# single element has been read.
|
|
56
|
+
raise ValueError(f"Output path is the input: {out}")
|
|
57
|
+
if out.exists() and not options.overwrite:
|
|
58
|
+
raise FileExistsError(
|
|
59
|
+
f"'{out}' already exists. Pass --overwrite to replace it."
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
console.print(f"[cyan]Converting[/] {file}: {plan.summary}")
|
|
63
|
+
for warning in plan.warnings:
|
|
64
|
+
console.print(f"[yellow]Warning:[/] {warning}")
|
|
65
|
+
|
|
66
|
+
with staged_outputs(plan.outputs) as staged:
|
|
67
|
+
converter.write(plan, staged, console)
|
|
68
|
+
|
|
69
|
+
for out in plan.outputs:
|
|
70
|
+
console.print(f"[green]Wrote[/] {out}")
|
|
71
|
+
return plan.outputs
|