pyadata-cli 0.6.1__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/PKG-INFO +17 -7
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/README.md +16 -6
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/pyproject.toml +1 -1
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/cli.py +292 -11
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/commands/__init__.py +1 -0
- pyadata_cli-0.7.0/src/adata/commands/convert.py +77 -0
- pyadata_cli-0.6.1/src/adata/commands/convert.py → pyadata_cli-0.7.0/src/adata/commands/recast.py +5 -5
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/commands/split.py +1 -29
- pyadata_cli-0.7.0/src/adata/conversion/__init__.py +186 -0
- pyadata_cli-0.7.0/src/adata/conversion/anndata_copy.py +90 -0
- pyadata_cli-0.7.0/src/adata/conversion/base.py +98 -0
- pyadata_cli-0.7.0/src/adata/conversion/tenx.py +812 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/core/concat.py +2 -2
- pyadata_cli-0.6.1/src/adata/core/convert.py → pyadata_cli-0.7.0/src/adata/core/recast.py +71 -16
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/core/subset.py +31 -34
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/storage/__init__.py +190 -12
- pyadata_cli-0.7.0/src/adata/util/names.py +69 -0
- pyadata_cli-0.7.0/src/adata/util/staging.py +63 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/LICENSE +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/__init__.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/commands/concat.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/commands/create.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/commands/export.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/commands/import_data.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/commands/info.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/commands/ls.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/commands/subset.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/core/__init__.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/core/info.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/core/query.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/core/read.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/core/select.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/elements/__init__.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/elements/read.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/elements/spec.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/elements/strings.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/elements/write.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/formats/__init__.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/formats/array.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/formats/common.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/formats/dataframe.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/formats/image.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/formats/json_data.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/formats/sparse.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/formats/validate.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/info.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/py.typed +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/read.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/util/__init__.py +0 -0
- {pyadata_cli-0.6.1 → pyadata_cli-0.7.0}/src/adata/util/path.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pyadata-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores
|
|
5
5
|
Keywords: anndata,h5ad,zarr,bioinformatics,single-cell,cli
|
|
6
6
|
Author: Aljes Binkevich
|
|
@@ -48,7 +48,7 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with
|
|
|
48
48
|
- Auto-detects `.h5ad` files vs `.zarr` directories
|
|
49
49
|
- Chunked processing for dense and sparse matrices (CSR/CSC)
|
|
50
50
|
- Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
|
|
51
|
-
- Converts between HDF5 and Zarr (v2 and v3) in either direction
|
|
51
|
+
- Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices
|
|
52
52
|
- Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
|
|
53
53
|
|
|
54
54
|
**Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
|
|
@@ -96,7 +96,8 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
|
|
|
96
96
|
- `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
|
|
97
97
|
- `split` – write one store per distinct value of an annotation column, with a CSV manifest.
|
|
98
98
|
- `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
|
|
99
|
-
- `convert` – change a
|
|
99
|
+
- `convert` – change a store's format: h5ad ↔ Zarr v2/v3, and 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) in and out, streaming, with the output written beside its destination and moved into place only once complete.
|
|
100
|
+
- `recast` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced. Called `convert` until 0.7.0.
|
|
100
101
|
- `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
|
|
101
102
|
- `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
|
|
102
103
|
|
|
@@ -122,16 +123,25 @@ adata concat per_sample/*.h5ad -o merged.h5ad --join outer --label sample
|
|
|
122
123
|
### Shrinking a store, and making encodings agree
|
|
123
124
|
|
|
124
125
|
```bash
|
|
125
|
-
adata
|
|
126
|
-
adata
|
|
127
|
-
adata
|
|
126
|
+
adata recast data.h5ad X --inplace --dtype float32
|
|
127
|
+
adata recast data.h5ad --all -o small.h5ad --dtype float32 --indices-dtype int32
|
|
128
|
+
adata recast data.h5ad X -o csc.h5ad --layout csc
|
|
128
129
|
```
|
|
129
130
|
|
|
130
|
-
Counts written as float64 halve with no loss — and `
|
|
131
|
+
Counts written as float64 halve with no loss — and `recast` proves that
|
|
131
132
|
before it writes, by casting every value and casting it back. `concat`
|
|
132
133
|
refuses inputs whose matrices disagree about CSR versus CSC; `--layout` is
|
|
133
134
|
how you make them agree.
|
|
134
135
|
|
|
136
|
+
### Changing format
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
adata convert data.h5ad -o data.zarr
|
|
140
|
+
adata convert data.zarr -o data.h5ad --compression gzip
|
|
141
|
+
adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
|
|
142
|
+
adata convert counts.h5ad -o for_cellbender.h5 --layer counts
|
|
143
|
+
```
|
|
144
|
+
|
|
135
145
|
## Documentation
|
|
136
146
|
|
|
137
147
|
- [Get started](docs/GET_STARTED.md) — a short tutorial
|
|
@@ -8,7 +8,7 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with
|
|
|
8
8
|
- Auto-detects `.h5ad` files vs `.zarr` directories
|
|
9
9
|
- Chunked processing for dense and sparse matrices (CSR/CSC)
|
|
10
10
|
- Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
|
|
11
|
-
- Converts between HDF5 and Zarr (v2 and v3) in either direction
|
|
11
|
+
- Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices
|
|
12
12
|
- Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
|
|
13
13
|
|
|
14
14
|
**Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
|
|
@@ -56,7 +56,8 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
|
|
|
56
56
|
- `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
|
|
57
57
|
- `split` – write one store per distinct value of an annotation column, with a CSV manifest.
|
|
58
58
|
- `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
|
|
59
|
-
- `convert` – change a
|
|
59
|
+
- `convert` – change a store's format: h5ad ↔ Zarr v2/v3, and 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) in and out, streaming, with the output written beside its destination and moved into place only once complete.
|
|
60
|
+
- `recast` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced. Called `convert` until 0.7.0.
|
|
60
61
|
- `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
|
|
61
62
|
- `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
|
|
62
63
|
|
|
@@ -82,16 +83,25 @@ adata concat per_sample/*.h5ad -o merged.h5ad --join outer --label sample
|
|
|
82
83
|
### Shrinking a store, and making encodings agree
|
|
83
84
|
|
|
84
85
|
```bash
|
|
85
|
-
adata
|
|
86
|
-
adata
|
|
87
|
-
adata
|
|
86
|
+
adata recast data.h5ad X --inplace --dtype float32
|
|
87
|
+
adata recast data.h5ad --all -o small.h5ad --dtype float32 --indices-dtype int32
|
|
88
|
+
adata recast data.h5ad X -o csc.h5ad --layout csc
|
|
88
89
|
```
|
|
89
90
|
|
|
90
|
-
Counts written as float64 halve with no loss — and `
|
|
91
|
+
Counts written as float64 halve with no loss — and `recast` proves that
|
|
91
92
|
before it writes, by casting every value and casting it back. `concat`
|
|
92
93
|
refuses inputs whose matrices disagree about CSR versus CSC; `--layout` is
|
|
93
94
|
how you make them agree.
|
|
94
95
|
|
|
96
|
+
### Changing format
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
adata convert data.h5ad -o data.zarr
|
|
100
|
+
adata convert data.zarr -o data.h5ad --compression gzip
|
|
101
|
+
adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
|
|
102
|
+
adata convert counts.h5ad -o for_cellbender.h5 --layer counts
|
|
103
|
+
```
|
|
104
|
+
|
|
95
105
|
## Documentation
|
|
96
106
|
|
|
97
107
|
- [Get started](docs/GET_STARTED.md) — a short tutorial
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# so the distribution is published as `pyadata-cli`. The import package
|
|
4
4
|
# and the command are both still `adata`.
|
|
5
5
|
name = "pyadata-cli"
|
|
6
|
-
version = "0.
|
|
6
|
+
version = "0.7.0"
|
|
7
7
|
description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores"
|
|
8
8
|
readme = "README.md"
|
|
9
9
|
requires-python = ">=3.12"
|
|
@@ -6,11 +6,13 @@ from typing import Optional, Sequence, List
|
|
|
6
6
|
from rich.console import Console
|
|
7
7
|
import typer
|
|
8
8
|
|
|
9
|
-
from adata.
|
|
9
|
+
from adata.conversion import FORMATS, Options
|
|
10
|
+
from adata.core.recast import LAYOUTS
|
|
10
11
|
from adata.commands import (
|
|
11
12
|
MERGE_CHOICES,
|
|
12
13
|
concat_stores,
|
|
13
14
|
convert_store,
|
|
15
|
+
recast_store,
|
|
14
16
|
create_store,
|
|
15
17
|
split_store,
|
|
16
18
|
list_store,
|
|
@@ -26,7 +28,7 @@ from adata.commands import export_image as export_image_cmd
|
|
|
26
28
|
|
|
27
29
|
app = typer.Typer(
|
|
28
30
|
help="Streaming CLI for huge AnnData .h5ad and .zarr stores "
|
|
29
|
-
"(view, ls, create, subset, split, concat, convert, export, import)."
|
|
31
|
+
"(view, ls, create, subset, split, concat, convert, recast, export, import)."
|
|
30
32
|
)
|
|
31
33
|
# Use stderr for status/progress to keep stdout clean for data output
|
|
32
34
|
# force_terminal=True ensures Rich output is visible even in non-TTY environments
|
|
@@ -481,8 +483,285 @@ def concat(
|
|
|
481
483
|
# ============================================================================
|
|
482
484
|
# CONVERT command
|
|
483
485
|
# ============================================================================
|
|
484
|
-
|
|
486
|
+
#: The flags `recast` has and format conversion does not. Until 1.0.0, any of
|
|
487
|
+
#: them on `convert` means the old meaning of the command, and it forwards.
|
|
488
|
+
_RECAST_FLAGS = {
|
|
489
|
+
"inplace": "--inplace",
|
|
490
|
+
"convert_all": "--all",
|
|
491
|
+
"dtype": "--dtype",
|
|
492
|
+
"indices_dtype": "--indices-dtype",
|
|
493
|
+
"layout": "--layout",
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
#: Flags that only shape a recast; given without one of the above they have
|
|
497
|
+
#: nothing to act on.
|
|
498
|
+
_RECAST_ONLY_FLAGS = {"in_memory": "--in-memory", "chunk_rows": "--chunk"}
|
|
499
|
+
|
|
500
|
+
#: Format-conversion flags, by parameter name, as `Options.given` spells them.
|
|
501
|
+
_CONVERT_FLAGS = {
|
|
502
|
+
"zarr_format": "--zarr-format",
|
|
503
|
+
"compression": "--compression",
|
|
504
|
+
"force": "--force",
|
|
505
|
+
"overwrite": "--overwrite",
|
|
506
|
+
"genome": "--genome",
|
|
507
|
+
"var_names": "--var-names",
|
|
508
|
+
"make_unique": "--make-unique",
|
|
509
|
+
"feature_type": "--feature-type",
|
|
510
|
+
"split_feature_types": "--split-feature-types",
|
|
511
|
+
"layer": "--layer",
|
|
512
|
+
"gene_ids_column": "--gene-ids-column",
|
|
513
|
+
"gene_names_column": "--gene-names-column",
|
|
514
|
+
"feature_types_column": "--feature-types-column",
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
#: Flags that only make sense for format conversion; with recast's flags,
|
|
518
|
+
#: they mean two commands' worth of request at once.
|
|
519
|
+
_FORMAT_ONLY_FLAGS = {
|
|
520
|
+
"from_format": "--from",
|
|
521
|
+
"to_format": "--to",
|
|
522
|
+
**{
|
|
523
|
+
name: flag
|
|
524
|
+
for name, flag in _CONVERT_FLAGS.items()
|
|
525
|
+
if flag not in ("--force", "--zarr-format")
|
|
526
|
+
},
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _given(ctx: typer.Context, flags: dict) -> List[str]:
|
|
531
|
+
"""The flags among `flags` that were actually typed, not defaulted."""
|
|
532
|
+
from click.core import ParameterSource
|
|
533
|
+
|
|
534
|
+
return [
|
|
535
|
+
flag
|
|
536
|
+
for name, flag in flags.items()
|
|
537
|
+
if ctx.get_parameter_source(name) not in (None, ParameterSource.DEFAULT)
|
|
538
|
+
]
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
@app.command("convert", context_settings={"allow_extra_args": True})
|
|
485
542
|
def convert(
|
|
543
|
+
ctx: typer.Context,
|
|
544
|
+
file: Path = typer.Argument(
|
|
545
|
+
...,
|
|
546
|
+
help="Input file or store",
|
|
547
|
+
exists=True,
|
|
548
|
+
readable=True,
|
|
549
|
+
dir_okay=True,
|
|
550
|
+
file_okay=True,
|
|
551
|
+
),
|
|
552
|
+
output: Optional[Path] = typer.Option(
|
|
553
|
+
None,
|
|
554
|
+
"--output",
|
|
555
|
+
"-o",
|
|
556
|
+
help="Output path; its extension picks the format (.h5ad, .zarr, .h5)",
|
|
557
|
+
dir_okay=True,
|
|
558
|
+
file_okay=True,
|
|
559
|
+
),
|
|
560
|
+
from_format: Optional[str] = typer.Option(
|
|
561
|
+
None,
|
|
562
|
+
"--from",
|
|
563
|
+
help=f"Input format, if it cannot be detected: {', '.join(FORMATS)}",
|
|
564
|
+
),
|
|
565
|
+
to_format: Optional[str] = typer.Option(
|
|
566
|
+
None,
|
|
567
|
+
"--to",
|
|
568
|
+
help=f"Output format, if the extension does not say: {', '.join(FORMATS)}",
|
|
569
|
+
),
|
|
570
|
+
zarr_format: Optional[int] = typer.Option(
|
|
571
|
+
None,
|
|
572
|
+
"--zarr-format",
|
|
573
|
+
help="Zarr spec version to write (defaults to the source store's, else 3)",
|
|
574
|
+
),
|
|
575
|
+
compression: Optional[str] = typer.Option(
|
|
576
|
+
None,
|
|
577
|
+
"--compression",
|
|
578
|
+
help="none, gzip or lzf for h5ad; none, gzip, zstd or blosc for zarr "
|
|
579
|
+
"(defaults to the source's)",
|
|
580
|
+
),
|
|
581
|
+
force: bool = typer.Option(
|
|
582
|
+
False,
|
|
583
|
+
"--force",
|
|
584
|
+
help="Convert despite a lossy or inflating result",
|
|
585
|
+
),
|
|
586
|
+
overwrite: bool = typer.Option(
|
|
587
|
+
False,
|
|
588
|
+
"--overwrite",
|
|
589
|
+
help="Replace an existing output",
|
|
590
|
+
),
|
|
591
|
+
genome: Optional[str] = typer.Option(
|
|
592
|
+
None,
|
|
593
|
+
"--genome",
|
|
594
|
+
help="10x input: the genome to keep (picks the group in a Cell Ranger 2 file)",
|
|
595
|
+
),
|
|
596
|
+
var_names: str = typer.Option(
|
|
597
|
+
"gene_symbols",
|
|
598
|
+
"--var-names",
|
|
599
|
+
help="10x input: name features by gene_symbols or gene_ids",
|
|
600
|
+
),
|
|
601
|
+
make_unique: bool = typer.Option(
|
|
602
|
+
False,
|
|
603
|
+
"--make-unique",
|
|
604
|
+
help="10x input: suffix repeated var names, as var_names_make_unique does",
|
|
605
|
+
),
|
|
606
|
+
feature_type: Optional[str] = typer.Option(
|
|
607
|
+
None,
|
|
608
|
+
"--feature-type",
|
|
609
|
+
help="10x input: keep only this feature type, e.g. 'Gene Expression'",
|
|
610
|
+
),
|
|
611
|
+
split_feature_types: bool = typer.Option(
|
|
612
|
+
False,
|
|
613
|
+
"--split-feature-types",
|
|
614
|
+
help="10x input: write one output per feature type, OUT.<type>.h5ad",
|
|
615
|
+
),
|
|
616
|
+
layer: Optional[str] = typer.Option(
|
|
617
|
+
None,
|
|
618
|
+
"--layer",
|
|
619
|
+
help="10x output: the matrix to write: X (default), raw/X, or a layer",
|
|
620
|
+
),
|
|
621
|
+
gene_ids_column: Optional[str] = typer.Option(
|
|
622
|
+
None,
|
|
623
|
+
"--gene-ids-column",
|
|
624
|
+
help="10x output: var column holding gene ids (default gene_ids)",
|
|
625
|
+
),
|
|
626
|
+
gene_names_column: Optional[str] = typer.Option(
|
|
627
|
+
None,
|
|
628
|
+
"--gene-names-column",
|
|
629
|
+
help="10x output: var column holding gene names (default: var_names)",
|
|
630
|
+
),
|
|
631
|
+
feature_types_column: Optional[str] = typer.Option(
|
|
632
|
+
None,
|
|
633
|
+
"--feature-types-column",
|
|
634
|
+
help="10x output: var column holding feature types (default feature_types)",
|
|
635
|
+
),
|
|
636
|
+
# Deprecated: recast's flags, so `convert` keeps working as it did until
|
|
637
|
+
# 1.0.0. Hidden from --help; the old matrix paths arrive in ctx.args.
|
|
638
|
+
inplace: bool = typer.Option(False, "--inplace", hidden=True),
|
|
639
|
+
convert_all: bool = typer.Option(False, "--all", hidden=True),
|
|
640
|
+
dtype: Optional[str] = typer.Option(None, "--dtype", hidden=True),
|
|
641
|
+
indices_dtype: Optional[str] = typer.Option(None, "--indices-dtype", hidden=True),
|
|
642
|
+
layout: Optional[str] = typer.Option(None, "--layout", hidden=True),
|
|
643
|
+
in_memory: bool = typer.Option(False, "--in-memory", hidden=True),
|
|
644
|
+
chunk_rows: int = typer.Option(1024, "--chunk", "-C", hidden=True),
|
|
645
|
+
) -> None:
|
|
646
|
+
"""
|
|
647
|
+
Convert a store to another format.
|
|
648
|
+
|
|
649
|
+
The output's extension picks the format: .h5ad, .zarr, or .h5 for a 10x
|
|
650
|
+
Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5). Between h5ad and
|
|
651
|
+
zarr every element is copied, streamed, keeping the source's chunking and
|
|
652
|
+
compression where the target can express them. A 10x matrix is read as
|
|
653
|
+
scanpy.read_10x_h5(gex_only=False) reads it, keeping integer counts, and
|
|
654
|
+
written as Cell Ranger writes it.
|
|
655
|
+
|
|
656
|
+
Nothing is written until every check has passed, and the output is moved
|
|
657
|
+
into place only once complete: a failed conversion leaves nothing
|
|
658
|
+
behind. An existing output is refused unless --overwrite is given.
|
|
659
|
+
|
|
660
|
+
To change a matrix's dtype, layout or density, use `recast`. Until 1.0.0,
|
|
661
|
+
`convert` given recast's flags still forwards to it, with a warning.
|
|
662
|
+
|
|
663
|
+
Examples:
|
|
664
|
+
adata convert data.h5ad -o data.zarr
|
|
665
|
+
adata convert data.zarr -o data.h5ad --compression gzip
|
|
666
|
+
adata convert data.zarr -o v3.zarr --zarr-format 3
|
|
667
|
+
adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad
|
|
668
|
+
adata convert raw_feature_bc_matrix.h5 -o rna.h5ad --feature-type "Gene Expression"
|
|
669
|
+
adata convert counts.h5ad -o for_cellbender.h5 --layer counts
|
|
670
|
+
"""
|
|
671
|
+
entries = list(ctx.args)
|
|
672
|
+
recast_flags = _given(ctx, _RECAST_FLAGS)
|
|
673
|
+
|
|
674
|
+
try:
|
|
675
|
+
if recast_flags:
|
|
676
|
+
mixed = _given(ctx, _FORMAT_ONLY_FLAGS)
|
|
677
|
+
if mixed:
|
|
678
|
+
raise ValueError(
|
|
679
|
+
f"{', '.join(mixed)} cannot be combined with "
|
|
680
|
+
f"{', '.join(recast_flags)}: the first change a store's "
|
|
681
|
+
"format, the second its matrices. Run `adata recast` for "
|
|
682
|
+
"the matrices."
|
|
683
|
+
)
|
|
684
|
+
console.print(
|
|
685
|
+
"[yellow]Warning:[/] 'convert' with "
|
|
686
|
+
f"{', '.join(recast_flags)} is deprecated and will be removed "
|
|
687
|
+
"in 1.0.0; use 'recast' instead.",
|
|
688
|
+
)
|
|
689
|
+
if not inplace and output is None:
|
|
690
|
+
raise ValueError("Output file is required. Use --output/-o or --inplace.")
|
|
691
|
+
if layout is not None and layout not in LAYOUTS:
|
|
692
|
+
raise ValueError(f"--layout must be one of: {', '.join(LAYOUTS)}.")
|
|
693
|
+
if zarr_format is not None and zarr_format not in (2, 3):
|
|
694
|
+
raise ValueError("--zarr-format must be 2 or 3.")
|
|
695
|
+
recast_store(
|
|
696
|
+
file=file,
|
|
697
|
+
entries=entries or None,
|
|
698
|
+
output=output,
|
|
699
|
+
console=console,
|
|
700
|
+
dtype=dtype,
|
|
701
|
+
indices_dtype=indices_dtype,
|
|
702
|
+
layout=layout,
|
|
703
|
+
convert_all=convert_all,
|
|
704
|
+
inplace=inplace,
|
|
705
|
+
chunk_rows=chunk_rows,
|
|
706
|
+
in_memory=in_memory,
|
|
707
|
+
force=force,
|
|
708
|
+
zarr_format=zarr_format,
|
|
709
|
+
)
|
|
710
|
+
return
|
|
711
|
+
|
|
712
|
+
if entries:
|
|
713
|
+
hint = (
|
|
714
|
+
f" To write it, pass -o {entries[0]}."
|
|
715
|
+
if output is None and len(entries) == 1
|
|
716
|
+
else ""
|
|
717
|
+
)
|
|
718
|
+
raise ValueError(
|
|
719
|
+
f"Unexpected argument{'s' if len(entries) > 1 else ''}: "
|
|
720
|
+
f"{' '.join(entries)}. convert takes one input.{hint} To change "
|
|
721
|
+
"a matrix's dtype or layout, use `adata recast`."
|
|
722
|
+
)
|
|
723
|
+
stray = _given(ctx, _RECAST_ONLY_FLAGS)
|
|
724
|
+
if stray:
|
|
725
|
+
raise ValueError(
|
|
726
|
+
f"{', '.join(stray)} {'applies' if len(stray) == 1 else 'apply'} "
|
|
727
|
+
"to `adata recast`, not to format conversion."
|
|
728
|
+
)
|
|
729
|
+
if output is None:
|
|
730
|
+
raise ValueError("Output path is required: pass -o/--output.")
|
|
731
|
+
|
|
732
|
+
convert_store(
|
|
733
|
+
file,
|
|
734
|
+
output,
|
|
735
|
+
console,
|
|
736
|
+
src_format=from_format,
|
|
737
|
+
dst_format=to_format,
|
|
738
|
+
options=Options(
|
|
739
|
+
given=frozenset(_given(ctx, _CONVERT_FLAGS)),
|
|
740
|
+
zarr_format=zarr_format,
|
|
741
|
+
compression=compression,
|
|
742
|
+
force=force,
|
|
743
|
+
overwrite=overwrite,
|
|
744
|
+
genome=genome,
|
|
745
|
+
var_names=var_names,
|
|
746
|
+
make_unique=make_unique,
|
|
747
|
+
feature_type=feature_type,
|
|
748
|
+
split_feature_types=split_feature_types,
|
|
749
|
+
layer=layer,
|
|
750
|
+
gene_ids_column=gene_ids_column,
|
|
751
|
+
gene_names_column=gene_names_column,
|
|
752
|
+
feature_types_column=feature_types_column,
|
|
753
|
+
),
|
|
754
|
+
)
|
|
755
|
+
except Exception as e:
|
|
756
|
+
console.print(f"[bold red]Error:[/] {e}")
|
|
757
|
+
raise typer.Exit(code=1)
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
# ============================================================================
|
|
761
|
+
# RECAST command
|
|
762
|
+
# ============================================================================
|
|
763
|
+
@app.command("recast")
|
|
764
|
+
def recast(
|
|
486
765
|
file: Path = typer.Argument(
|
|
487
766
|
...,
|
|
488
767
|
help="Input .h5ad/.zarr",
|
|
@@ -493,7 +772,7 @@ def convert(
|
|
|
493
772
|
),
|
|
494
773
|
entries: Optional[List[str]] = typer.Argument(
|
|
495
774
|
None,
|
|
496
|
-
help="Matrix paths to
|
|
775
|
+
help="Matrix paths to recast, e.g. 'X', 'layers/counts', 'raw/X'",
|
|
497
776
|
),
|
|
498
777
|
output: Optional[Path] = typer.Option(
|
|
499
778
|
None,
|
|
@@ -511,7 +790,7 @@ def convert(
|
|
|
511
790
|
convert_all: bool = typer.Option(
|
|
512
791
|
False,
|
|
513
792
|
"--all",
|
|
514
|
-
help="
|
|
793
|
+
help="Recast X, every layer, and raw/X",
|
|
515
794
|
),
|
|
516
795
|
dtype: Optional[str] = typer.Option(
|
|
517
796
|
None,
|
|
@@ -531,7 +810,7 @@ def convert(
|
|
|
531
810
|
force: bool = typer.Option(
|
|
532
811
|
False,
|
|
533
812
|
"--force",
|
|
534
|
-
help="
|
|
813
|
+
help="Recast despite a lossy cast or a large size increase",
|
|
535
814
|
),
|
|
536
815
|
in_memory: bool = typer.Option(
|
|
537
816
|
False,
|
|
@@ -562,11 +841,13 @@ def convert(
|
|
|
562
841
|
both. Transposing streams by default, so it works on matrices too large
|
|
563
842
|
to load.
|
|
564
843
|
|
|
844
|
+
This was `convert` until 0.7.0; `convert` now changes a store's format.
|
|
845
|
+
|
|
565
846
|
Examples:
|
|
566
|
-
adata
|
|
567
|
-
adata
|
|
568
|
-
adata
|
|
569
|
-
adata
|
|
847
|
+
adata recast data.h5ad X -o out.h5ad --dtype float32
|
|
848
|
+
adata recast data.h5ad X -o out.h5ad --layout csc
|
|
849
|
+
adata recast data.h5ad X --inplace --dtype float32 --indices-dtype int32
|
|
850
|
+
adata recast data.h5ad --all -o out.h5ad --dtype float32
|
|
570
851
|
"""
|
|
571
852
|
if not inplace and output is None:
|
|
572
853
|
console.print(
|
|
@@ -586,7 +867,7 @@ def convert(
|
|
|
586
867
|
raise typer.Exit(code=1)
|
|
587
868
|
|
|
588
869
|
try:
|
|
589
|
-
|
|
870
|
+
recast_store(
|
|
590
871
|
file=file,
|
|
591
872
|
entries=list(entries) if entries else None,
|
|
592
873
|
output=output,
|
|
@@ -6,4 +6,5 @@ from adata.commands.ls import list_store
|
|
|
6
6
|
from adata.commands.create import create_store
|
|
7
7
|
from adata.commands.split import split_store
|
|
8
8
|
from adata.commands.concat import MERGE_CHOICES, MERGE_STRATEGIES, concat_stores
|
|
9
|
+
from adata.commands.recast import recast_store
|
|
9
10
|
from adata.commands.convert import convert_store
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""The `convert` command: rewrite a store in another format.
|
|
2
|
+
|
|
3
|
+
Every refusal happens while building the plan, before an output exists, and
|
|
4
|
+
the outputs are written beside their destination and moved into place only
|
|
5
|
+
once complete. A conversion that fails therefore leaves nothing behind, and
|
|
6
|
+
never a half-written store that the next command would read as a whole one.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import List, Optional
|
|
13
|
+
|
|
14
|
+
from rich.console import Console
|
|
15
|
+
|
|
16
|
+
from adata.conversion import (
|
|
17
|
+
Options,
|
|
18
|
+
check_options,
|
|
19
|
+
converter_for,
|
|
20
|
+
detect_format,
|
|
21
|
+
output_format,
|
|
22
|
+
)
|
|
23
|
+
from adata.util.staging import staged_outputs
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _same_path(left: Path, right: Path) -> bool:
|
|
27
|
+
try:
|
|
28
|
+
return left.resolve() == right.resolve()
|
|
29
|
+
except OSError:
|
|
30
|
+
return False
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def convert_store(
|
|
34
|
+
file: Path,
|
|
35
|
+
output: Path,
|
|
36
|
+
console: Console,
|
|
37
|
+
*,
|
|
38
|
+
src_format: Optional[str] = None,
|
|
39
|
+
dst_format: Optional[str] = None,
|
|
40
|
+
options: Options = Options(),
|
|
41
|
+
) -> List[Path]:
|
|
42
|
+
"""Convert `file` into `output`; return the paths written."""
|
|
43
|
+
file, output = Path(file), Path(output)
|
|
44
|
+
if str(output) == "-":
|
|
45
|
+
raise ValueError(
|
|
46
|
+
"convert writes a store, which cannot go to stdout; give a path "
|
|
47
|
+
"with -o."
|
|
48
|
+
)
|
|
49
|
+
if options.zarr_format not in (None, 2, 3):
|
|
50
|
+
raise ValueError("--zarr-format must be 2 or 3.")
|
|
51
|
+
|
|
52
|
+
src_format = src_format or detect_format(file)
|
|
53
|
+
dst_format = output_format(output, dst_format)
|
|
54
|
+
converter = converter_for(src_format, dst_format)
|
|
55
|
+
check_options(converter, options, src_format, dst_format)
|
|
56
|
+
|
|
57
|
+
plan = converter.plan(file, output, src_format, dst_format, options)
|
|
58
|
+
for out in plan.outputs:
|
|
59
|
+
if _same_path(file, out):
|
|
60
|
+
# Writing a Zarr store over its own source empties it before a
|
|
61
|
+
# single element has been read.
|
|
62
|
+
raise ValueError(f"Output path is the input: {out}")
|
|
63
|
+
if out.exists() and not options.overwrite:
|
|
64
|
+
raise FileExistsError(
|
|
65
|
+
f"'{out}' already exists. Pass --overwrite to replace it."
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
console.print(f"[cyan]Converting[/] {file}: {plan.summary}")
|
|
69
|
+
for warning in plan.warnings:
|
|
70
|
+
console.print(f"[yellow]Warning:[/] {warning}")
|
|
71
|
+
|
|
72
|
+
with staged_outputs(plan.outputs) as staged:
|
|
73
|
+
converter.write(plan, staged, console)
|
|
74
|
+
|
|
75
|
+
for out in plan.outputs:
|
|
76
|
+
console.print(f"[green]Wrote[/] {out}")
|
|
77
|
+
return plan.outputs
|
pyadata_cli-0.6.1/src/adata/commands/convert.py → pyadata_cli-0.7.0/src/adata/commands/recast.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""The `
|
|
1
|
+
"""The `recast` command: rewrite matrices, copy everything else.
|
|
2
2
|
|
|
3
3
|
Writes a whole new store rather than editing one in place, for the same
|
|
4
4
|
reason `subset` does: a conversion that fails half way leaves the original
|
|
@@ -14,7 +14,7 @@ from typing import Any, List, Optional
|
|
|
14
14
|
|
|
15
15
|
from rich.console import Console
|
|
16
16
|
|
|
17
|
-
from adata.core.
|
|
17
|
+
from adata.core.recast import (
|
|
18
18
|
DEFAULT_CHUNK,
|
|
19
19
|
LAYOUTS,
|
|
20
20
|
Plan,
|
|
@@ -61,7 +61,7 @@ def _resolve(root: Any, path: str) -> Any:
|
|
|
61
61
|
return obj
|
|
62
62
|
|
|
63
63
|
|
|
64
|
-
def
|
|
64
|
+
def recast_store(
|
|
65
65
|
file: Path,
|
|
66
66
|
entries: Optional[List[str]],
|
|
67
67
|
output: Optional[Path],
|
|
@@ -109,9 +109,9 @@ def convert_store(
|
|
|
109
109
|
backend = detect_backend(file)
|
|
110
110
|
if backend == "zarr":
|
|
111
111
|
base = file.stem if file.suffix else file.name
|
|
112
|
-
dst_path = file.with_name(f"{base}.
|
|
112
|
+
dst_path = file.with_name(f"{base}.recast-tmp.zarr")
|
|
113
113
|
else:
|
|
114
|
-
dst_path = file.with_name(f"{file.name}.
|
|
114
|
+
dst_path = file.with_name(f"{file.name}.recast-tmp")
|
|
115
115
|
if dst_path.exists():
|
|
116
116
|
raise FileExistsError(f"Temporary path already exists: {dst_path}")
|
|
117
117
|
else:
|
|
@@ -9,7 +9,6 @@ on is read, and each output is produced by the same subset machinery used by
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import csv
|
|
12
|
-
import re
|
|
13
12
|
from pathlib import Path
|
|
14
13
|
from typing import Dict, List, Optional, Tuple
|
|
15
14
|
|
|
@@ -19,34 +18,7 @@ from rich.console import Console
|
|
|
19
18
|
from adata.core.select import group_indices
|
|
20
19
|
from adata.core.subset import split_h5ad
|
|
21
20
|
from adata.storage import detect_backend, open_store
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def sanitize_label(label: str) -> str:
|
|
25
|
-
"""Turn a column value into something safe to use as a filename."""
|
|
26
|
-
text = (label or "").strip()
|
|
27
|
-
if not text:
|
|
28
|
-
return "NA"
|
|
29
|
-
text = text.replace("/", "_").replace("\\", "_")
|
|
30
|
-
text = re.sub(r"\s+", "_", text)
|
|
31
|
-
text = re.sub(r"[^A-Za-z0-9_.-]+", "", text)
|
|
32
|
-
return text or "NA"
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
def unique_names(labels: List[str]) -> Dict[str, str]:
|
|
36
|
-
"""Map each label to a distinct filename stem.
|
|
37
|
-
|
|
38
|
-
Sanitising can map two different labels onto the same stem ("a/b" and
|
|
39
|
-
"a b"), so collisions get a numeric suffix rather than silently
|
|
40
|
-
overwriting one another.
|
|
41
|
-
"""
|
|
42
|
-
used: Dict[str, int] = {}
|
|
43
|
-
mapping: Dict[str, str] = {}
|
|
44
|
-
for label in labels:
|
|
45
|
-
base = sanitize_label(label)
|
|
46
|
-
count = used.get(base, 0)
|
|
47
|
-
used[base] = count + 1
|
|
48
|
-
mapping[label] = base if count == 0 else f"{base}_{count}"
|
|
49
|
-
return mapping
|
|
21
|
+
from adata.util.names import unique_names
|
|
50
22
|
|
|
51
23
|
|
|
52
24
|
def split_store(
|