pyadata-cli 0.7.1__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/PKG-INFO +4 -3
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/README.md +3 -2
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/pyproject.toml +1 -1
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/cli.py +18 -11
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/import_data.py +3 -1
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/conversion/__init__.py +25 -4
- pyadata_cli-0.8.0/src/adata/conversion/features.py +380 -0
- pyadata_cli-0.8.0/src/adata/conversion/mtx.py +605 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/conversion/tenx.py +41 -275
- pyadata_cli-0.8.0/src/adata/core/csr_build.py +328 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/recast.py +19 -102
- pyadata_cli-0.8.0/src/adata/formats/mtx.py +503 -0
- pyadata_cli-0.8.0/src/adata/formats/sparse.py +152 -0
- pyadata_cli-0.8.0/src/adata/formats/textio.py +236 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/storage/__init__.py +2 -10
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/util/path.py +12 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/util/staging.py +5 -2
- pyadata_cli-0.7.1/src/adata/formats/sparse.py +0 -253
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/LICENSE +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/__init__.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/__init__.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/concat.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/convert.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/create.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/export.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/info.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/ls.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/recast.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/split.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/subset.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/conversion/anndata_copy.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/conversion/base.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/__init__.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/concat.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/info.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/query.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/read.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/select.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/subset.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/elements/__init__.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/elements/read.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/elements/spec.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/elements/strings.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/elements/write.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/__init__.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/array.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/common.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/dataframe.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/image.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/json_data.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/validate.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/info.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/py.typed +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/read.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/util/__init__.py +0 -0
- {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/util/names.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pyadata-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores
|
|
5
5
|
Keywords: anndata,h5ad,zarr,bioinformatics,single-cell,cli
|
|
6
6
|
Author: Aljes Binkevich
|
|
@@ -48,7 +48,7 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with
|
|
|
48
48
|
- Auto-detects `.h5ad` files vs `.zarr` directories
|
|
49
49
|
- Chunked processing for dense and sparse matrices (CSR/CSC)
|
|
50
50
|
- Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
|
|
51
|
-
- Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices
|
|
51
|
+
- Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices and 10x / STARsolo MTX folders
|
|
52
52
|
- Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
|
|
53
53
|
|
|
54
54
|
**Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
|
|
@@ -96,7 +96,7 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
|
|
|
96
96
|
- `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
|
|
97
97
|
- `split` – write one store per distinct value of an annotation column, with a CSV manifest.
|
|
98
98
|
- `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
|
|
99
|
-
- `convert` – change a store's format: h5ad ↔ Zarr v2/v3,
|
|
99
|
+
- `convert` – change a store's format: h5ad ↔ Zarr v2/v3, 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) and 10x / STARsolo MTX folders in and out, streaming, with the output written beside its destination and moved into place only once complete.
|
|
100
100
|
- `recast` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced. Called `convert` until 0.7.0.
|
|
101
101
|
- `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
|
|
102
102
|
- `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
|
|
@@ -140,6 +140,7 @@ adata convert data.h5ad -o data.zarr
|
|
|
140
140
|
adata convert data.zarr -o data.h5ad --compression gzip
|
|
141
141
|
adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
|
|
142
142
|
adata convert counts.h5ad -o for_cellbender.h5 --layer counts
|
|
143
|
+
adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
|
|
143
144
|
```
|
|
144
145
|
|
|
145
146
|
## Documentation
|
|
@@ -8,7 +8,7 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with
|
|
|
8
8
|
- Auto-detects `.h5ad` files vs `.zarr` directories
|
|
9
9
|
- Chunked processing for dense and sparse matrices (CSR/CSC)
|
|
10
10
|
- Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
|
|
11
|
-
- Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices
|
|
11
|
+
- Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices and 10x / STARsolo MTX folders
|
|
12
12
|
- Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
|
|
13
13
|
|
|
14
14
|
**Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
|
|
@@ -56,7 +56,7 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
|
|
|
56
56
|
- `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
|
|
57
57
|
- `split` – write one store per distinct value of an annotation column, with a CSV manifest.
|
|
58
58
|
- `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
|
|
59
|
-
- `convert` – change a store's format: h5ad ↔ Zarr v2/v3,
|
|
59
|
+
- `convert` – change a store's format: h5ad ↔ Zarr v2/v3, 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) and 10x / STARsolo MTX folders in and out, streaming, with the output written beside its destination and moved into place only once complete.
|
|
60
60
|
- `recast` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced. Called `convert` until 0.7.0.
|
|
61
61
|
- `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
|
|
62
62
|
- `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
|
|
@@ -100,6 +100,7 @@ adata convert data.h5ad -o data.zarr
|
|
|
100
100
|
adata convert data.zarr -o data.h5ad --compression gzip
|
|
101
101
|
adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
|
|
102
102
|
adata convert counts.h5ad -o for_cellbender.h5 --layer counts
|
|
103
|
+
adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
|
|
103
104
|
```
|
|
104
105
|
|
|
105
106
|
## Documentation
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# so the distribution is published as `pyadata-cli`. The import package
|
|
4
4
|
# and the command are both still `adata`.
|
|
5
5
|
name = "pyadata-cli"
|
|
6
|
-
version = "0.
|
|
6
|
+
version = "0.8.0"
|
|
7
7
|
description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores"
|
|
8
8
|
readme = "README.md"
|
|
9
9
|
requires-python = ">=3.12"
|
|
@@ -553,7 +553,7 @@ def convert(
|
|
|
553
553
|
ctx: typer.Context,
|
|
554
554
|
file: Path = typer.Argument(
|
|
555
555
|
...,
|
|
556
|
-
help="Input file or
|
|
556
|
+
help="Input file, store, or 10x/STARsolo matrix folder",
|
|
557
557
|
exists=True,
|
|
558
558
|
readable=True,
|
|
559
559
|
dir_okay=True,
|
|
@@ -563,7 +563,8 @@ def convert(
|
|
|
563
563
|
None,
|
|
564
564
|
"--output",
|
|
565
565
|
"-o",
|
|
566
|
-
help="Output path; its extension picks the format (.h5ad, .zarr, .h5)"
|
|
566
|
+
help="Output path; its extension picks the format (.h5ad, .zarr, .h5), "
|
|
567
|
+
"or a folder with --to 10x-mtx",
|
|
567
568
|
dir_okay=True,
|
|
568
569
|
file_okay=True,
|
|
569
570
|
),
|
|
@@ -586,7 +587,7 @@ def convert(
|
|
|
586
587
|
None,
|
|
587
588
|
"--compression",
|
|
588
589
|
help="none, gzip or lzf for h5ad; none, gzip, zstd or blosc for zarr "
|
|
589
|
-
"(defaults to the source's)",
|
|
590
|
+
"(defaults to the source's); gzip or none for a 10x-mtx folder",
|
|
590
591
|
),
|
|
591
592
|
force: bool = typer.Option(
|
|
592
593
|
False,
|
|
@@ -601,7 +602,8 @@ def convert(
|
|
|
601
602
|
genome: Optional[str] = typer.Option(
|
|
602
603
|
None,
|
|
603
604
|
"--genome",
|
|
604
|
-
help="10x input: the genome to keep (picks the group
|
|
605
|
+
help="10x input: the genome to keep (picks the group or subfolder of a "
|
|
606
|
+
"Cell Ranger 2 matrix)",
|
|
605
607
|
),
|
|
606
608
|
var_names: str = typer.Option(
|
|
607
609
|
"gene_symbols",
|
|
@@ -657,11 +659,13 @@ def convert(
|
|
|
657
659
|
Convert a store to another format.
|
|
658
660
|
|
|
659
661
|
The output's extension picks the format: .h5ad, .zarr, or .h5 for a 10x
|
|
660
|
-
Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5)
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
662
|
+
Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5); --to 10x-mtx
|
|
663
|
+
writes a Matrix Market folder. Between h5ad and zarr every element is
|
|
664
|
+
copied, streamed, keeping the source's chunking and compression where the
|
|
665
|
+
target can express them. A 10x matrix is read as scanpy's read_10x_h5 or
|
|
666
|
+
read_10x_mtx (gex_only=False) reads it, keeping integer counts, and
|
|
667
|
+
written as Cell Ranger writes it. MTX folders from Cell Ranger and
|
|
668
|
+
STARsolo, velocyto's included, are read in two streaming passes.
|
|
665
669
|
|
|
666
670
|
Nothing is written until every check has passed, and the output is moved
|
|
667
671
|
into place only once complete: a failed conversion leaves nothing
|
|
@@ -677,6 +681,9 @@ def convert(
|
|
|
677
681
|
adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad
|
|
678
682
|
adata convert raw_feature_bc_matrix.h5 -o rna.h5ad --feature-type "Gene Expression"
|
|
679
683
|
adata convert counts.h5ad -o for_cellbender.h5 --layer counts
|
|
684
|
+
adata convert Solo.out/Gene/filtered -o counts.h5ad
|
|
685
|
+
adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
|
|
686
|
+
adata convert counts.h5ad -o counts_mtx --to 10x-mtx --layer counts
|
|
680
687
|
"""
|
|
681
688
|
entries = list(ctx.args)
|
|
682
689
|
recast_flags = _given(ctx, _RECAST_FLAGS)
|
|
@@ -1117,7 +1124,7 @@ def export_sparse(
|
|
|
1117
1124
|
"--output",
|
|
1118
1125
|
"-o",
|
|
1119
1126
|
writable=True,
|
|
1120
|
-
help="Output .mtx file path (defaults to stdout)",
|
|
1127
|
+
help="Output .mtx file path; .mtx.gz gzips it (defaults to stdout)",
|
|
1121
1128
|
),
|
|
1122
1129
|
head: Optional[int] = typer.Option(
|
|
1123
1130
|
None, "--head", "-n", help="Output only the first n entries of mtx file"
|
|
@@ -1407,7 +1414,7 @@ def import_sparse(
|
|
|
1407
1414
|
..., help="Object path to create/replace (e.g., 'X', 'layers/counts')"
|
|
1408
1415
|
),
|
|
1409
1416
|
input_file: Path = typer.Argument(
|
|
1410
|
-
..., help="Input .mtx file", exists=True, readable=True
|
|
1417
|
+
..., help="Input .mtx or .mtx.gz file", exists=True, readable=True
|
|
1411
1418
|
),
|
|
1412
1419
|
output: Optional[Path] = typer.Option(
|
|
1413
1420
|
None,
|
|
@@ -60,7 +60,9 @@ def import_object(
|
|
|
60
60
|
console: Console,
|
|
61
61
|
) -> None:
|
|
62
62
|
target_file = _prepare_target_path(file, output_file, inplace, console)
|
|
63
|
-
|
|
63
|
+
suffixes = [x.lower() for x in input_file.suffixes]
|
|
64
|
+
# A gzipped matrix is still a matrix: `counts.mtx.gz` imports as `.mtx`.
|
|
65
|
+
ext = suffixes[-2] if len(suffixes) > 1 and suffixes[-1] == ".gz" else input_file.suffix.lower()
|
|
64
66
|
|
|
65
67
|
if ext not in EXTENSION_FORMAT:
|
|
66
68
|
raise ValueError(
|
|
@@ -17,7 +17,7 @@ from typing import Dict, FrozenSet, List, Optional
|
|
|
17
17
|
|
|
18
18
|
import h5py
|
|
19
19
|
|
|
20
|
-
from adata.conversion import tenx
|
|
20
|
+
from adata.conversion import mtx, tenx
|
|
21
21
|
from adata.conversion.anndata_copy import AnnDataCopy
|
|
22
22
|
from adata.conversion.base import ConversionPlan, Converter, Format, Options
|
|
23
23
|
from adata.storage import has_valid_anndata_root_attrs, is_zarr_path
|
|
@@ -74,6 +74,15 @@ FORMATS: Dict[str, Format] = {
|
|
|
74
74
|
importer=tenx.TenxImport(),
|
|
75
75
|
exporter=tenx.TenxExport(),
|
|
76
76
|
),
|
|
77
|
+
# A folder, so no extension: chosen as output with `--to 10x-mtx`.
|
|
78
|
+
"10x-mtx": Format(
|
|
79
|
+
name="10x-mtx",
|
|
80
|
+
description="10x Genomics / STARsolo Matrix Market folder",
|
|
81
|
+
extensions=(),
|
|
82
|
+
sniff=mtx.sniff,
|
|
83
|
+
importer=mtx.MtxImport(),
|
|
84
|
+
exporter=mtx.MtxExport(),
|
|
85
|
+
),
|
|
77
86
|
}
|
|
78
87
|
|
|
79
88
|
_ANNDATA_COPY = AnnDataCopy()
|
|
@@ -106,6 +115,12 @@ def detect_format(path: Path) -> str:
|
|
|
106
115
|
for fmt in FORMATS.values():
|
|
107
116
|
if (fmt.anndata or fmt.sniff is None) and suffix in fmt.extensions:
|
|
108
117
|
return fmt.name
|
|
118
|
+
if suffix in (".mtx", ".gz") and path.name.lower().endswith((".mtx", ".mtx.gz")):
|
|
119
|
+
raise ValueError(
|
|
120
|
+
f"'{path.name}' is a single Matrix Market file, which `convert` "
|
|
121
|
+
"does not read on its own: give the folder holding it with its "
|
|
122
|
+
"barcodes and features, or use `adata import sparse`."
|
|
123
|
+
)
|
|
109
124
|
raise ValueError(
|
|
110
125
|
f"Cannot tell what format '{path}' is. Pass --from, one of: {_names()}."
|
|
111
126
|
)
|
|
@@ -130,15 +145,21 @@ def output_format(path: Path, requested: Optional[str] = None) -> str:
|
|
|
130
145
|
if by_extension is None:
|
|
131
146
|
raise ValueError(
|
|
132
147
|
f"Cannot tell the output format from '{path.name}'. "
|
|
133
|
-
f"Use a known extension or pass --to, one of: {_names()}
|
|
148
|
+
f"Use a known extension or pass --to, one of: {_names()} "
|
|
149
|
+
"(--to 10x-mtx for a folder)."
|
|
134
150
|
)
|
|
135
151
|
return by_extension
|
|
136
152
|
|
|
137
153
|
if by_extension is not None and by_extension != requested:
|
|
154
|
+
extensions = FORMATS[requested].extensions
|
|
138
155
|
raise ValueError(
|
|
139
156
|
f"--to {requested} contradicts the output's extension, which means "
|
|
140
|
-
f"{by_extension}. Name the output
|
|
141
|
-
|
|
157
|
+
f"{by_extension}. Name the output "
|
|
158
|
+
+ (
|
|
159
|
+
f"with one of: {', '.join(extensions)}."
|
|
160
|
+
if extensions
|
|
161
|
+
else "as a folder, without an extension."
|
|
162
|
+
)
|
|
142
163
|
)
|
|
143
164
|
if (requested == "zarr") != (suffix == ".zarr"):
|
|
144
165
|
raise ValueError(
|
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
"""The feature table shared by every 10x-style format.
|
|
2
|
+
|
|
3
|
+
A 10x HDF5 matrix and a 10x MTX folder record the same things about each
|
|
4
|
+
feature -- an id, a name, a type -- and import and export them the same way.
|
|
5
|
+
What is common lives here, so `tenx` and `mtx` differ only in how they read
|
|
6
|
+
and write their files:
|
|
7
|
+
|
|
8
|
+
* import: which features each output keeps (`plan_targets`), how they are
|
|
9
|
+
named (`annotate`), and how obs and var are written (`write_obs_index`,
|
|
10
|
+
`write_var`);
|
|
11
|
+
* export: which matrix to write (`plan_matrix_source`) and each feature's id,
|
|
12
|
+
name and type, worked out from var (`export_features`).
|
|
13
|
+
|
|
14
|
+
The AnnData produced matches what scanpy's `read_10x_h5` and `read_10x_mtx`
|
|
15
|
+
give with `gex_only=False`.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
from adata.conversion.base import Options
|
|
27
|
+
from adata.core.recast import describe
|
|
28
|
+
from adata.elements import spec
|
|
29
|
+
from adata.elements.read import (
|
|
30
|
+
dataframe_columns,
|
|
31
|
+
element_len,
|
|
32
|
+
read_str_all,
|
|
33
|
+
resolve_index,
|
|
34
|
+
)
|
|
35
|
+
from adata.elements.strings import string_dtype_for
|
|
36
|
+
from adata.elements.write import (
|
|
37
|
+
write_categorical,
|
|
38
|
+
write_dataframe_header,
|
|
39
|
+
write_dense,
|
|
40
|
+
write_string_array,
|
|
41
|
+
)
|
|
42
|
+
from adata.storage import create_dataset, is_group, is_zarr_group, open_store
|
|
43
|
+
from adata.util.names import count_duplicates, make_index_unique, unique_names
|
|
44
|
+
|
|
45
|
+
VAR_NAMES = ("gene_symbols", "gene_ids")
|
|
46
|
+
|
|
47
|
+
#: Rows of a string column handled per read when streaming one.
|
|
48
|
+
STRING_STEP = 1 << 18
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# ---------------------------------------------------------------------------
|
|
52
|
+
# import
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class Target:
|
|
57
|
+
"""One AnnData to write: which features it keeps, and their annotation."""
|
|
58
|
+
|
|
59
|
+
label: Optional[str]
|
|
60
|
+
keep: Optional[np.ndarray]
|
|
61
|
+
var_names: List[str]
|
|
62
|
+
columns: Dict[str, np.ndarray]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def type_counts(types: np.ndarray) -> List[Tuple[str, int]]:
|
|
66
|
+
"""Feature types in order of first appearance, with how many of each."""
|
|
67
|
+
labels, first, counts = np.unique(types, return_index=True, return_counts=True)
|
|
68
|
+
order = np.argsort(first)
|
|
69
|
+
return [(str(labels[i]), int(counts[i])) for i in order]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def check_import_options(options: Options) -> None:
|
|
73
|
+
if options.var_names not in VAR_NAMES:
|
|
74
|
+
raise ValueError(f"--var-names must be one of: {', '.join(VAR_NAMES)}.")
|
|
75
|
+
if options.feature_type is not None and options.split_feature_types:
|
|
76
|
+
raise ValueError(
|
|
77
|
+
"--feature-type keeps one type and --split-feature-types writes "
|
|
78
|
+
"every type separately; pass one or the other."
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def annotate(
|
|
83
|
+
names: np.ndarray,
|
|
84
|
+
columns: Dict[str, np.ndarray],
|
|
85
|
+
keep: Optional[np.ndarray],
|
|
86
|
+
options: Options,
|
|
87
|
+
label: Optional[str],
|
|
88
|
+
) -> Target:
|
|
89
|
+
"""var for the features in `keep`: their names, and their columns."""
|
|
90
|
+
|
|
91
|
+
def pick(values: np.ndarray) -> np.ndarray:
|
|
92
|
+
return values if keep is None else values[keep]
|
|
93
|
+
|
|
94
|
+
picked = {key: pick(values) for key, values in columns.items()}
|
|
95
|
+
if options.var_names == "gene_ids":
|
|
96
|
+
# As scanpy's read_10x_mtx does it: ids become the names, and the
|
|
97
|
+
# symbols move into a column where the ids were.
|
|
98
|
+
index = list(picked["gene_ids"])
|
|
99
|
+
picked = {
|
|
100
|
+
("gene_symbols" if key == "gene_ids" else key): (
|
|
101
|
+
pick(names) if key == "gene_ids" else values
|
|
102
|
+
)
|
|
103
|
+
for key, values in picked.items()
|
|
104
|
+
}
|
|
105
|
+
else:
|
|
106
|
+
index = list(pick(names))
|
|
107
|
+
if options.make_unique:
|
|
108
|
+
index = make_index_unique(index)
|
|
109
|
+
return Target(label=label, keep=keep, var_names=index, columns=picked)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def plan_targets(
|
|
113
|
+
output: Path,
|
|
114
|
+
names: np.ndarray,
|
|
115
|
+
columns: Dict[str, np.ndarray],
|
|
116
|
+
types: Optional[np.ndarray],
|
|
117
|
+
keep: np.ndarray,
|
|
118
|
+
options: Options,
|
|
119
|
+
*,
|
|
120
|
+
noun: str,
|
|
121
|
+
) -> Tuple[List[Path], List[Target], List[str]]:
|
|
122
|
+
"""The outputs to write, the features each keeps, and any warnings.
|
|
123
|
+
|
|
124
|
+
One output, unless `--split-feature-types` asks for one per type. `keep`
|
|
125
|
+
is what survives filters the caller applied first, such as `--genome`;
|
|
126
|
+
`noun` names the input in messages ("file", "folder").
|
|
127
|
+
"""
|
|
128
|
+
groups: List[Tuple[Optional[str], np.ndarray]] = []
|
|
129
|
+
if options.feature_type is not None:
|
|
130
|
+
available = [t for t, _ in type_counts(types)]
|
|
131
|
+
if options.feature_type not in available:
|
|
132
|
+
raise ValueError(
|
|
133
|
+
f"No features of type {options.feature_type!r}; this {noun} "
|
|
134
|
+
f"has: {', '.join(available)}."
|
|
135
|
+
)
|
|
136
|
+
groups.append((options.feature_type, keep & (types == options.feature_type)))
|
|
137
|
+
elif options.split_feature_types:
|
|
138
|
+
for label, _ in type_counts(types[keep]):
|
|
139
|
+
groups.append((label, keep & (types == label)))
|
|
140
|
+
else:
|
|
141
|
+
groups.append((None, keep))
|
|
142
|
+
|
|
143
|
+
stems = unique_names([label for label, _ in groups if label is not None])
|
|
144
|
+
outputs: List[Path] = []
|
|
145
|
+
targets: List[Target] = []
|
|
146
|
+
warnings: List[str] = []
|
|
147
|
+
for label, mask in groups:
|
|
148
|
+
path = output
|
|
149
|
+
if options.split_feature_types:
|
|
150
|
+
path = output.with_name(f"{output.stem}.{stems[label]}{output.suffix}")
|
|
151
|
+
selection = None if mask.all() else np.flatnonzero(mask)
|
|
152
|
+
target = annotate(names, columns, selection, options, label)
|
|
153
|
+
duplicated = count_duplicates(target.var_names)
|
|
154
|
+
if duplicated:
|
|
155
|
+
where = f" in {path.name}" if options.split_feature_types else ""
|
|
156
|
+
warnings.append(
|
|
157
|
+
f"{duplicated:,} var names{where} repeat an earlier one. "
|
|
158
|
+
"Pass --make-unique to suffix them as anndata's "
|
|
159
|
+
"var_names_make_unique does."
|
|
160
|
+
)
|
|
161
|
+
outputs.append(path)
|
|
162
|
+
targets.append(target)
|
|
163
|
+
return outputs, targets, warnings
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def describe_import(
|
|
167
|
+
n_barcodes: int,
|
|
168
|
+
n_features: int,
|
|
169
|
+
types: Optional[np.ndarray],
|
|
170
|
+
targets: Sequence[Target],
|
|
171
|
+
) -> str:
|
|
172
|
+
"""`1,222 barcodes x 33,538 features (Gene Expression 33,538)`, and so on."""
|
|
173
|
+
described = f"{n_barcodes:,} barcodes x {n_features:,} features"
|
|
174
|
+
if types is not None:
|
|
175
|
+
described += " (" + ", ".join(f"{t} {n:,}" for t, n in type_counts(types)) + ")"
|
|
176
|
+
if len(targets) > 1:
|
|
177
|
+
described += f", into {len(targets)} outputs by feature type"
|
|
178
|
+
elif targets[0].keep is not None:
|
|
179
|
+
described += f", keeping {len(targets[0].keep):,} features"
|
|
180
|
+
return described
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def write_obs_index(root: Any, n: int, blocks: Iterable[Sequence[str]]) -> Any:
|
|
184
|
+
"""obs, holding only its index: the barcodes, written a block at a time.
|
|
185
|
+
|
|
186
|
+
Returns the group, for the caller to add columns to. A source that gives
|
|
187
|
+
more or fewer than `n` barcodes is refused.
|
|
188
|
+
"""
|
|
189
|
+
obs = root.create_group("obs")
|
|
190
|
+
spec.set_encoding(obs, spec.DATAFRAME)
|
|
191
|
+
obs.attrs["_index"] = "_index"
|
|
192
|
+
|
|
193
|
+
backend = "zarr" if is_zarr_group(root) else "hdf5"
|
|
194
|
+
index = create_dataset(obs, "_index", shape=(n,), dtype=string_dtype_for(backend))
|
|
195
|
+
written = 0
|
|
196
|
+
for block in blocks:
|
|
197
|
+
if written + len(block) > n:
|
|
198
|
+
raise ValueError(f"There are more than {n:,} barcodes.")
|
|
199
|
+
index[written : written + len(block)] = np.asarray(block, dtype=object)
|
|
200
|
+
written += len(block)
|
|
201
|
+
if written != n:
|
|
202
|
+
raise ValueError(f"There are {written:,} barcodes, not {n:,}.")
|
|
203
|
+
spec.set_encoding(index, spec.STRING_ARRAY)
|
|
204
|
+
return obs
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def write_var(root: Any, target: Target) -> None:
|
|
208
|
+
"""var, with string columns encoded as anndata's writer would encode them.
|
|
209
|
+
|
|
210
|
+
anndata turns any string column with a repeated value into a categorical
|
|
211
|
+
when it writes (`strings_to_categoricals`), so an h5ad scanpy wrote from
|
|
212
|
+
the same file holds `feature_types` and `genome` as categoricals. Doing
|
|
213
|
+
the same keeps the two interchangeable, and those columns a few bytes
|
|
214
|
+
instead of a string per feature.
|
|
215
|
+
"""
|
|
216
|
+
var = write_dataframe_header(root, "var", target.var_names, list(target.columns))
|
|
217
|
+
for key, values in target.columns.items():
|
|
218
|
+
if values.dtype.kind == "b":
|
|
219
|
+
write_dense(var, key, values)
|
|
220
|
+
continue
|
|
221
|
+
categories, codes = np.unique(values, return_inverse=True)
|
|
222
|
+
if len(categories) < len(values):
|
|
223
|
+
write_categorical(var, key, codes, list(categories))
|
|
224
|
+
else:
|
|
225
|
+
write_string_array(var, key, values)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
# ---------------------------------------------------------------------------
|
|
229
|
+
# export
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def matrix_path(layer: Optional[str]) -> str:
|
|
233
|
+
"""`--layer` as a path: X, raw/X, a layer's name, or layers/NAME."""
|
|
234
|
+
if layer is None or layer in ("X", "raw/X") or layer.startswith("layers/"):
|
|
235
|
+
return layer or "X"
|
|
236
|
+
return f"layers/{layer}"
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def resolve(root: Any, path: str) -> Any:
|
|
240
|
+
node = root
|
|
241
|
+
for part in path.split("/"):
|
|
242
|
+
if not is_group(node) or part not in node:
|
|
243
|
+
raise KeyError(f"{path!r} not found in the store.")
|
|
244
|
+
node = node[part]
|
|
245
|
+
return node
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
@dataclass
|
|
249
|
+
class ExportFeatures:
|
|
250
|
+
ids: List[str]
|
|
251
|
+
names: List[str]
|
|
252
|
+
types: List[str]
|
|
253
|
+
genomes: List[str]
|
|
254
|
+
tags: Dict[str, List[str]]
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
@dataclass
|
|
258
|
+
class MatrixSource:
|
|
259
|
+
"""The matrix an export writes, and what is known about it before writing."""
|
|
260
|
+
|
|
261
|
+
path: str
|
|
262
|
+
n_obs: int
|
|
263
|
+
n_var: int
|
|
264
|
+
dtype: np.dtype
|
|
265
|
+
features: ExportFeatures
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def plan_matrix_source(
|
|
269
|
+
source: Path, options: Options, warnings: List[str]
|
|
270
|
+
) -> MatrixSource:
|
|
271
|
+
"""Check the matrix `--layer` names against obs and var; read var's features.
|
|
272
|
+
|
|
273
|
+
Warns when the matrix holds floats, which 10x formats are not expected to.
|
|
274
|
+
"""
|
|
275
|
+
path = matrix_path(options.layer)
|
|
276
|
+
with open_store(source, "r", require_anndata=False) as store:
|
|
277
|
+
root = store.root
|
|
278
|
+
matrix = describe(resolve(root, path))
|
|
279
|
+
var_path = "raw/var" if path.startswith("raw/") else "var"
|
|
280
|
+
var, obs = resolve(root, var_path), resolve(root, "obs")
|
|
281
|
+
obs_index, _ = resolve_index(obs, "obs")
|
|
282
|
+
var_index, _ = resolve_index(var, "var")
|
|
283
|
+
n_obs, n_var = element_len(obs_index), element_len(var_index)
|
|
284
|
+
if matrix.shape != (n_obs, n_var):
|
|
285
|
+
raise ValueError(
|
|
286
|
+
f"{path} is {matrix.shape[0]} x {matrix.shape[1]}, but obs has "
|
|
287
|
+
f"{n_obs} and {var_path} has {n_var} entries."
|
|
288
|
+
)
|
|
289
|
+
features = export_features(root, var, var_index, options, warnings)
|
|
290
|
+
|
|
291
|
+
if matrix.dtype.kind == "f":
|
|
292
|
+
warnings.append(
|
|
293
|
+
f"{path} holds {matrix.dtype} values. Cell Ranger writes integer "
|
|
294
|
+
"counts, and tools that read its output may expect them: export "
|
|
295
|
+
"a counts layer with --layer, or recast to int32 first."
|
|
296
|
+
)
|
|
297
|
+
return MatrixSource(
|
|
298
|
+
path=path, n_obs=n_obs, n_var=n_var, dtype=matrix.dtype, features=features
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def export_features(
|
|
303
|
+
root: Any,
|
|
304
|
+
var: Any,
|
|
305
|
+
var_index: Any,
|
|
306
|
+
options: Options,
|
|
307
|
+
warnings: List[str],
|
|
308
|
+
) -> ExportFeatures:
|
|
309
|
+
"""Work out each feature's id, name, type and genome from var."""
|
|
310
|
+
columns = dataframe_columns(var)
|
|
311
|
+
n = element_len(var_index)
|
|
312
|
+
|
|
313
|
+
def column(name: str) -> List[str]:
|
|
314
|
+
return read_str_all(var[name], parent_group=var)
|
|
315
|
+
|
|
316
|
+
def requested(flag: str, name: Optional[str]) -> Optional[str]:
|
|
317
|
+
if name is not None and name not in columns:
|
|
318
|
+
raise ValueError(
|
|
319
|
+
f"{flag} {name}: no such var column. It has: "
|
|
320
|
+
f"{', '.join(columns) or 'none'}."
|
|
321
|
+
)
|
|
322
|
+
return name
|
|
323
|
+
|
|
324
|
+
ids_col = requested("--gene-ids-column", options.gene_ids_column)
|
|
325
|
+
names_col = requested("--gene-names-column", options.gene_names_column)
|
|
326
|
+
types_col = requested("--feature-types-column", options.feature_types_column)
|
|
327
|
+
index = read_str_all(var_index)
|
|
328
|
+
|
|
329
|
+
missing = []
|
|
330
|
+
if ids_col is None and names_col is None and "gene_ids" not in columns and "gene_symbols" in columns:
|
|
331
|
+
# var_names are ids already, as `--var-names gene_ids` leaves them.
|
|
332
|
+
ids, names = index, column("gene_symbols")
|
|
333
|
+
else:
|
|
334
|
+
names = column(names_col) if names_col else index
|
|
335
|
+
if ids_col is None and "gene_ids" in columns:
|
|
336
|
+
ids_col = "gene_ids"
|
|
337
|
+
if ids_col is not None:
|
|
338
|
+
ids = column(ids_col)
|
|
339
|
+
else:
|
|
340
|
+
missing.append("gene ids (a gene_ids column, or --gene-ids-column)")
|
|
341
|
+
ids = names
|
|
342
|
+
if types_col is None and "feature_types" in columns:
|
|
343
|
+
types_col = "feature_types"
|
|
344
|
+
if types_col is not None:
|
|
345
|
+
types = column(types_col)
|
|
346
|
+
else:
|
|
347
|
+
missing.append("feature types (a feature_types column, or --feature-types-column)")
|
|
348
|
+
types = ["Gene Expression"] * n
|
|
349
|
+
|
|
350
|
+
if missing:
|
|
351
|
+
if not options.force:
|
|
352
|
+
raise ValueError(
|
|
353
|
+
f"var has no {' and no '.join(missing)}, which a 10x matrix "
|
|
354
|
+
"records for every feature. Pass --force to write placeholders: "
|
|
355
|
+
"the names as ids, and 'Gene Expression' as the type."
|
|
356
|
+
)
|
|
357
|
+
warnings.append(f"Writing placeholders for {' and '.join(missing)}.")
|
|
358
|
+
|
|
359
|
+
genomes = column("genome") if "genome" in columns else [""] * n
|
|
360
|
+
tags: Dict[str, List[str]] = {}
|
|
361
|
+
for key in recorded_tag_keys(root):
|
|
362
|
+
if key != "genome" and key in columns:
|
|
363
|
+
tags[key] = column(key)
|
|
364
|
+
return ExportFeatures(ids=ids, names=names, types=types, genomes=genomes, tags=tags)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def cellranger_meta(root: Any) -> Optional[Any]:
|
|
368
|
+
"""`uns/cellranger`, as an import recorded it, if there is one."""
|
|
369
|
+
if "uns" not in root or "cellranger" not in root["uns"]:
|
|
370
|
+
return None
|
|
371
|
+
meta = root["uns"]["cellranger"]
|
|
372
|
+
return meta if is_group(meta) else None
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def recorded_tag_keys(root: Any) -> List[str]:
|
|
376
|
+
meta = cellranger_meta(root)
|
|
377
|
+
if meta is None or "feature_tag_keys" not in meta:
|
|
378
|
+
return []
|
|
379
|
+
return read_str_all(meta["feature_tag_keys"])
|
|
380
|
+
|