pyadata-cli 0.7.1__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/PKG-INFO +4 -3
  2. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/README.md +3 -2
  3. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/pyproject.toml +1 -1
  4. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/cli.py +18 -11
  5. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/import_data.py +3 -1
  6. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/conversion/__init__.py +25 -4
  7. pyadata_cli-0.8.0/src/adata/conversion/features.py +380 -0
  8. pyadata_cli-0.8.0/src/adata/conversion/mtx.py +605 -0
  9. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/conversion/tenx.py +41 -275
  10. pyadata_cli-0.8.0/src/adata/core/csr_build.py +328 -0
  11. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/recast.py +19 -102
  12. pyadata_cli-0.8.0/src/adata/formats/mtx.py +503 -0
  13. pyadata_cli-0.8.0/src/adata/formats/sparse.py +152 -0
  14. pyadata_cli-0.8.0/src/adata/formats/textio.py +236 -0
  15. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/storage/__init__.py +2 -10
  16. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/util/path.py +12 -0
  17. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/util/staging.py +5 -2
  18. pyadata_cli-0.7.1/src/adata/formats/sparse.py +0 -253
  19. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/LICENSE +0 -0
  20. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/__init__.py +0 -0
  21. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/__init__.py +0 -0
  22. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/concat.py +0 -0
  23. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/convert.py +0 -0
  24. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/create.py +0 -0
  25. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/export.py +0 -0
  26. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/info.py +0 -0
  27. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/ls.py +0 -0
  28. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/recast.py +0 -0
  29. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/split.py +0 -0
  30. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/commands/subset.py +0 -0
  31. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/conversion/anndata_copy.py +0 -0
  32. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/conversion/base.py +0 -0
  33. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/__init__.py +0 -0
  34. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/concat.py +0 -0
  35. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/info.py +0 -0
  36. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/query.py +0 -0
  37. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/read.py +0 -0
  38. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/select.py +0 -0
  39. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/core/subset.py +0 -0
  40. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/elements/__init__.py +0 -0
  41. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/elements/read.py +0 -0
  42. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/elements/spec.py +0 -0
  43. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/elements/strings.py +0 -0
  44. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/elements/write.py +0 -0
  45. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/__init__.py +0 -0
  46. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/array.py +0 -0
  47. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/common.py +0 -0
  48. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/dataframe.py +0 -0
  49. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/image.py +0 -0
  50. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/json_data.py +0 -0
  51. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/formats/validate.py +0 -0
  52. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/info.py +0 -0
  53. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/py.typed +0 -0
  54. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/read.py +0 -0
  55. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/util/__init__.py +0 -0
  56. {pyadata_cli-0.7.1 → pyadata_cli-0.8.0}/src/adata/util/names.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pyadata-cli
3
- Version: 0.7.1
3
+ Version: 0.8.0
4
4
  Summary: Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores
5
5
  Keywords: anndata,h5ad,zarr,bioinformatics,single-cell,cli
6
6
  Author: Aljes Binkevich
@@ -48,7 +48,7 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with
48
48
  - Auto-detects `.h5ad` files vs `.zarr` directories
49
49
  - Chunked processing for dense and sparse matrices (CSR/CSC)
50
50
  - Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
51
- - Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices
51
+ - Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices and 10x / STARsolo MTX folders
52
52
  - Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
53
53
 
54
54
  **Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
@@ -96,7 +96,7 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
96
96
  - `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
97
97
  - `split` – write one store per distinct value of an annotation column, with a CSV manifest.
98
98
  - `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
99
- - `convert` – change a store's format: h5ad ↔ Zarr v2/v3, and 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) in and out, streaming, with the output written beside its destination and moved into place only once complete.
99
+ - `convert` – change a store's format: h5ad ↔ Zarr v2/v3, 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) and 10x / STARsolo MTX folders in and out, streaming, with the output written beside its destination and moved into place only once complete.
100
100
  - `recast` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced. Called `convert` until 0.7.0.
101
101
  - `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
102
102
  - `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
@@ -140,6 +140,7 @@ adata convert data.h5ad -o data.zarr
140
140
  adata convert data.zarr -o data.h5ad --compression gzip
141
141
  adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
142
142
  adata convert counts.h5ad -o for_cellbender.h5 --layer counts
143
+ adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
143
144
  ```
144
145
 
145
146
  ## Documentation
@@ -8,7 +8,7 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with
8
8
  - Auto-detects `.h5ad` files vs `.zarr` directories
9
9
  - Chunked processing for dense and sparse matrices (CSR/CSC)
10
10
  - Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one
11
- - Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices
11
+ - Converts between HDF5 and Zarr (v2 and v3) in either direction with `adata convert`, and to and from 10x Genomics HDF5 matrices and 10x / STARsolo MTX folders
12
12
  - Rich terminal output with progress indicators, kept on stderr so results pipe cleanly
13
13
 
14
14
  **Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)**
@@ -56,7 +56,7 @@ Run help at any level (e.g. `adata --help`, `adata export --help`).
56
56
  - `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`).
57
57
  - `split` – write one store per distinct value of an annotation column, with a CSV manifest.
58
58
  - `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns.
59
- - `convert` – change a store's format: h5ad ↔ Zarr v2/v3, and 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) in and out, streaming, with the output written beside its destination and moved into place only once complete.
59
+ - `convert` – change a store's format: h5ad ↔ Zarr v2/v3, 10x Genomics HDF5 (Cell Ranger's `*_feature_bc_matrix.h5`) and 10x / STARsolo MTX folders in and out, streaming, with the output written beside its destination and moved into place only once complete.
60
60
  - `recast` – change a matrix's dtype, layout (CSR/CSC/dense) or density, streaming; refuses a lossy cast or a large size increase unless forced. Called `convert` until 0.7.0.
61
61
  - `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given.
62
62
  - `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF).
@@ -100,6 +100,7 @@ adata convert data.h5ad -o data.zarr
100
100
  adata convert data.zarr -o data.h5ad --compression gzip
101
101
  adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
102
102
  adata convert counts.h5ad -o for_cellbender.h5 --layer counts
103
+ adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
103
104
  ```
104
105
 
105
106
  ## Documentation
@@ -3,7 +3,7 @@
3
3
  # so the distribution is published as `pyadata-cli`. The import package
4
4
  # and the command are both still `adata`.
5
5
  name = "pyadata-cli"
6
- version = "0.7.1"
6
+ version = "0.8.0"
7
7
  description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores"
8
8
  readme = "README.md"
9
9
  requires-python = ">=3.12"
@@ -553,7 +553,7 @@ def convert(
553
553
  ctx: typer.Context,
554
554
  file: Path = typer.Argument(
555
555
  ...,
556
- help="Input file or store",
556
+ help="Input file, store, or 10x/STARsolo matrix folder",
557
557
  exists=True,
558
558
  readable=True,
559
559
  dir_okay=True,
@@ -563,7 +563,8 @@ def convert(
563
563
  None,
564
564
  "--output",
565
565
  "-o",
566
- help="Output path; its extension picks the format (.h5ad, .zarr, .h5)",
566
+ help="Output path; its extension picks the format (.h5ad, .zarr, .h5), "
567
+ "or a folder with --to 10x-mtx",
567
568
  dir_okay=True,
568
569
  file_okay=True,
569
570
  ),
@@ -586,7 +587,7 @@ def convert(
586
587
  None,
587
588
  "--compression",
588
589
  help="none, gzip or lzf for h5ad; none, gzip, zstd or blosc for zarr "
589
- "(defaults to the source's)",
590
+ "(defaults to the source's); gzip or none for a 10x-mtx folder",
590
591
  ),
591
592
  force: bool = typer.Option(
592
593
  False,
@@ -601,7 +602,8 @@ def convert(
601
602
  genome: Optional[str] = typer.Option(
602
603
  None,
603
604
  "--genome",
604
- help="10x input: the genome to keep (picks the group in a Cell Ranger 2 file)",
605
+ help="10x input: the genome to keep (picks the group or subfolder of a "
606
+ "Cell Ranger 2 matrix)",
605
607
  ),
606
608
  var_names: str = typer.Option(
607
609
  "gene_symbols",
@@ -657,11 +659,13 @@ def convert(
657
659
  Convert a store to another format.
658
660
 
659
661
  The output's extension picks the format: .h5ad, .zarr, or .h5 for a 10x
660
- Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5). Between h5ad and
661
- zarr every element is copied, streamed, keeping the source's chunking and
662
- compression where the target can express them. A 10x matrix is read as
663
- scanpy.read_10x_h5(gex_only=False) reads it, keeping integer counts, and
664
- written as Cell Ranger writes it.
662
+ Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5); --to 10x-mtx
663
+ writes a Matrix Market folder. Between h5ad and zarr every element is
664
+ copied, streamed, keeping the source's chunking and compression where the
665
+ target can express them. A 10x matrix is read as scanpy's read_10x_h5 or
666
+ read_10x_mtx (gex_only=False) reads it, keeping integer counts, and
667
+ written as Cell Ranger writes it. MTX folders from Cell Ranger and
668
+ STARsolo, velocyto's included, are read in two streaming passes.
665
669
 
666
670
  Nothing is written until every check has passed, and the output is moved
667
671
  into place only once complete: a failed conversion leaves nothing
@@ -677,6 +681,9 @@ def convert(
677
681
  adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad
678
682
  adata convert raw_feature_bc_matrix.h5 -o rna.h5ad --feature-type "Gene Expression"
679
683
  adata convert counts.h5ad -o for_cellbender.h5 --layer counts
684
+ adata convert Solo.out/Gene/filtered -o counts.h5ad
685
+ adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
686
+ adata convert counts.h5ad -o counts_mtx --to 10x-mtx --layer counts
680
687
  """
681
688
  entries = list(ctx.args)
682
689
  recast_flags = _given(ctx, _RECAST_FLAGS)
@@ -1117,7 +1124,7 @@ def export_sparse(
1117
1124
  "--output",
1118
1125
  "-o",
1119
1126
  writable=True,
1120
- help="Output .mtx file path (defaults to stdout)",
1127
+ help="Output .mtx file path; .mtx.gz gzips it (defaults to stdout)",
1121
1128
  ),
1122
1129
  head: Optional[int] = typer.Option(
1123
1130
  None, "--head", "-n", help="Output only the first n entries of mtx file"
@@ -1407,7 +1414,7 @@ def import_sparse(
1407
1414
  ..., help="Object path to create/replace (e.g., 'X', 'layers/counts')"
1408
1415
  ),
1409
1416
  input_file: Path = typer.Argument(
1410
- ..., help="Input .mtx file", exists=True, readable=True
1417
+ ..., help="Input .mtx or .mtx.gz file", exists=True, readable=True
1411
1418
  ),
1412
1419
  output: Optional[Path] = typer.Option(
1413
1420
  None,
@@ -60,7 +60,9 @@ def import_object(
60
60
  console: Console,
61
61
  ) -> None:
62
62
  target_file = _prepare_target_path(file, output_file, inplace, console)
63
- ext = input_file.suffix.lower()
63
+ suffixes = [x.lower() for x in input_file.suffixes]
64
+ # A gzipped matrix is still a matrix: `counts.mtx.gz` imports as `.mtx`.
65
+ ext = suffixes[-2] if len(suffixes) > 1 and suffixes[-1] == ".gz" else input_file.suffix.lower()
64
66
 
65
67
  if ext not in EXTENSION_FORMAT:
66
68
  raise ValueError(
@@ -17,7 +17,7 @@ from typing import Dict, FrozenSet, List, Optional
17
17
 
18
18
  import h5py
19
19
 
20
- from adata.conversion import tenx
20
+ from adata.conversion import mtx, tenx
21
21
  from adata.conversion.anndata_copy import AnnDataCopy
22
22
  from adata.conversion.base import ConversionPlan, Converter, Format, Options
23
23
  from adata.storage import has_valid_anndata_root_attrs, is_zarr_path
@@ -74,6 +74,15 @@ FORMATS: Dict[str, Format] = {
74
74
  importer=tenx.TenxImport(),
75
75
  exporter=tenx.TenxExport(),
76
76
  ),
77
+ # A folder, so no extension: chosen as output with `--to 10x-mtx`.
78
+ "10x-mtx": Format(
79
+ name="10x-mtx",
80
+ description="10x Genomics / STARsolo Matrix Market folder",
81
+ extensions=(),
82
+ sniff=mtx.sniff,
83
+ importer=mtx.MtxImport(),
84
+ exporter=mtx.MtxExport(),
85
+ ),
77
86
  }
78
87
 
79
88
  _ANNDATA_COPY = AnnDataCopy()
@@ -106,6 +115,12 @@ def detect_format(path: Path) -> str:
106
115
  for fmt in FORMATS.values():
107
116
  if (fmt.anndata or fmt.sniff is None) and suffix in fmt.extensions:
108
117
  return fmt.name
118
+ if suffix in (".mtx", ".gz") and path.name.lower().endswith((".mtx", ".mtx.gz")):
119
+ raise ValueError(
120
+ f"'{path.name}' is a single Matrix Market file, which `convert` "
121
+ "does not read on its own: give the folder holding it with its "
122
+ "barcodes and features, or use `adata import sparse`."
123
+ )
109
124
  raise ValueError(
110
125
  f"Cannot tell what format '{path}' is. Pass --from, one of: {_names()}."
111
126
  )
@@ -130,15 +145,21 @@ def output_format(path: Path, requested: Optional[str] = None) -> str:
130
145
  if by_extension is None:
131
146
  raise ValueError(
132
147
  f"Cannot tell the output format from '{path.name}'. "
133
- f"Use a known extension or pass --to, one of: {_names()}."
148
+ f"Use a known extension or pass --to, one of: {_names()} "
149
+ "(--to 10x-mtx for a folder)."
134
150
  )
135
151
  return by_extension
136
152
 
137
153
  if by_extension is not None and by_extension != requested:
154
+ extensions = FORMATS[requested].extensions
138
155
  raise ValueError(
139
156
  f"--to {requested} contradicts the output's extension, which means "
140
- f"{by_extension}. Name the output with one of: "
141
- f"{', '.join(FORMATS[requested].extensions)}."
157
+ f"{by_extension}. Name the output "
158
+ + (
159
+ f"with one of: {', '.join(extensions)}."
160
+ if extensions
161
+ else "as a folder, without an extension."
162
+ )
142
163
  )
143
164
  if (requested == "zarr") != (suffix == ".zarr"):
144
165
  raise ValueError(
@@ -0,0 +1,380 @@
1
+ """The feature table shared by every 10x-style format.
2
+
3
+ A 10x HDF5 matrix and a 10x MTX folder record the same things about each
4
+ feature -- an id, a name, a type -- and import and export them the same way.
5
+ What is common lives here, so `tenx` and `mtx` differ only in how they read
6
+ and write their files:
7
+
8
+ * import: which features each output keeps (`plan_targets`), how they are
9
+ named (`annotate`), and how obs and var are written (`write_obs_index`,
10
+ `write_var`);
11
+ * export: which matrix to write (`plan_matrix_source`) and each feature's id,
12
+ name and type, worked out from var (`export_features`).
13
+
14
+ The AnnData produced matches what scanpy's `read_10x_h5` and `read_10x_mtx`
15
+ give with `gex_only=False`.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ from dataclasses import dataclass
21
+ from pathlib import Path
22
+ from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
23
+
24
+ import numpy as np
25
+
26
+ from adata.conversion.base import Options
27
+ from adata.core.recast import describe
28
+ from adata.elements import spec
29
+ from adata.elements.read import (
30
+ dataframe_columns,
31
+ element_len,
32
+ read_str_all,
33
+ resolve_index,
34
+ )
35
+ from adata.elements.strings import string_dtype_for
36
+ from adata.elements.write import (
37
+ write_categorical,
38
+ write_dataframe_header,
39
+ write_dense,
40
+ write_string_array,
41
+ )
42
+ from adata.storage import create_dataset, is_group, is_zarr_group, open_store
43
+ from adata.util.names import count_duplicates, make_index_unique, unique_names
44
+
45
+ VAR_NAMES = ("gene_symbols", "gene_ids")
46
+
47
+ #: Rows of a string column handled per read when streaming one.
48
+ STRING_STEP = 1 << 18
49
+
50
+
51
+ # ---------------------------------------------------------------------------
52
+ # import
53
+
54
+
55
+ @dataclass
56
+ class Target:
57
+ """One AnnData to write: which features it keeps, and their annotation."""
58
+
59
+ label: Optional[str]
60
+ keep: Optional[np.ndarray]
61
+ var_names: List[str]
62
+ columns: Dict[str, np.ndarray]
63
+
64
+
65
+ def type_counts(types: np.ndarray) -> List[Tuple[str, int]]:
66
+ """Feature types in order of first appearance, with how many of each."""
67
+ labels, first, counts = np.unique(types, return_index=True, return_counts=True)
68
+ order = np.argsort(first)
69
+ return [(str(labels[i]), int(counts[i])) for i in order]
70
+
71
+
72
+ def check_import_options(options: Options) -> None:
73
+ if options.var_names not in VAR_NAMES:
74
+ raise ValueError(f"--var-names must be one of: {', '.join(VAR_NAMES)}.")
75
+ if options.feature_type is not None and options.split_feature_types:
76
+ raise ValueError(
77
+ "--feature-type keeps one type and --split-feature-types writes "
78
+ "every type separately; pass one or the other."
79
+ )
80
+
81
+
82
+ def annotate(
83
+ names: np.ndarray,
84
+ columns: Dict[str, np.ndarray],
85
+ keep: Optional[np.ndarray],
86
+ options: Options,
87
+ label: Optional[str],
88
+ ) -> Target:
89
+ """var for the features in `keep`: their names, and their columns."""
90
+
91
+ def pick(values: np.ndarray) -> np.ndarray:
92
+ return values if keep is None else values[keep]
93
+
94
+ picked = {key: pick(values) for key, values in columns.items()}
95
+ if options.var_names == "gene_ids":
96
+ # As scanpy's read_10x_mtx does it: ids become the names, and the
97
+ # symbols move into a column where the ids were.
98
+ index = list(picked["gene_ids"])
99
+ picked = {
100
+ ("gene_symbols" if key == "gene_ids" else key): (
101
+ pick(names) if key == "gene_ids" else values
102
+ )
103
+ for key, values in picked.items()
104
+ }
105
+ else:
106
+ index = list(pick(names))
107
+ if options.make_unique:
108
+ index = make_index_unique(index)
109
+ return Target(label=label, keep=keep, var_names=index, columns=picked)
110
+
111
+
112
+ def plan_targets(
113
+ output: Path,
114
+ names: np.ndarray,
115
+ columns: Dict[str, np.ndarray],
116
+ types: Optional[np.ndarray],
117
+ keep: np.ndarray,
118
+ options: Options,
119
+ *,
120
+ noun: str,
121
+ ) -> Tuple[List[Path], List[Target], List[str]]:
122
+ """The outputs to write, the features each keeps, and any warnings.
123
+
124
+ One output, unless `--split-feature-types` asks for one per type. `keep`
125
+ is what survives filters the caller applied first, such as `--genome`;
126
+ `noun` names the input in messages ("file", "folder").
127
+ """
128
+ groups: List[Tuple[Optional[str], np.ndarray]] = []
129
+ if options.feature_type is not None:
130
+ available = [t for t, _ in type_counts(types)]
131
+ if options.feature_type not in available:
132
+ raise ValueError(
133
+ f"No features of type {options.feature_type!r}; this {noun} "
134
+ f"has: {', '.join(available)}."
135
+ )
136
+ groups.append((options.feature_type, keep & (types == options.feature_type)))
137
+ elif options.split_feature_types:
138
+ for label, _ in type_counts(types[keep]):
139
+ groups.append((label, keep & (types == label)))
140
+ else:
141
+ groups.append((None, keep))
142
+
143
+ stems = unique_names([label for label, _ in groups if label is not None])
144
+ outputs: List[Path] = []
145
+ targets: List[Target] = []
146
+ warnings: List[str] = []
147
+ for label, mask in groups:
148
+ path = output
149
+ if options.split_feature_types:
150
+ path = output.with_name(f"{output.stem}.{stems[label]}{output.suffix}")
151
+ selection = None if mask.all() else np.flatnonzero(mask)
152
+ target = annotate(names, columns, selection, options, label)
153
+ duplicated = count_duplicates(target.var_names)
154
+ if duplicated:
155
+ where = f" in {path.name}" if options.split_feature_types else ""
156
+ warnings.append(
157
+ f"{duplicated:,} var names{where} repeat an earlier one. "
158
+ "Pass --make-unique to suffix them as anndata's "
159
+ "var_names_make_unique does."
160
+ )
161
+ outputs.append(path)
162
+ targets.append(target)
163
+ return outputs, targets, warnings
164
+
165
+
166
+ def describe_import(
167
+ n_barcodes: int,
168
+ n_features: int,
169
+ types: Optional[np.ndarray],
170
+ targets: Sequence[Target],
171
+ ) -> str:
172
+ """`1,222 barcodes x 33,538 features (Gene Expression 33,538)`, and so on."""
173
+ described = f"{n_barcodes:,} barcodes x {n_features:,} features"
174
+ if types is not None:
175
+ described += " (" + ", ".join(f"{t} {n:,}" for t, n in type_counts(types)) + ")"
176
+ if len(targets) > 1:
177
+ described += f", into {len(targets)} outputs by feature type"
178
+ elif targets[0].keep is not None:
179
+ described += f", keeping {len(targets[0].keep):,} features"
180
+ return described
181
+
182
+
183
+ def write_obs_index(root: Any, n: int, blocks: Iterable[Sequence[str]]) -> Any:
184
+ """obs, holding only its index: the barcodes, written a block at a time.
185
+
186
+ Returns the group, for the caller to add columns to. A source that gives
187
+ more or fewer than `n` barcodes is refused.
188
+ """
189
+ obs = root.create_group("obs")
190
+ spec.set_encoding(obs, spec.DATAFRAME)
191
+ obs.attrs["_index"] = "_index"
192
+
193
+ backend = "zarr" if is_zarr_group(root) else "hdf5"
194
+ index = create_dataset(obs, "_index", shape=(n,), dtype=string_dtype_for(backend))
195
+ written = 0
196
+ for block in blocks:
197
+ if written + len(block) > n:
198
+ raise ValueError(f"There are more than {n:,} barcodes.")
199
+ index[written : written + len(block)] = np.asarray(block, dtype=object)
200
+ written += len(block)
201
+ if written != n:
202
+ raise ValueError(f"There are {written:,} barcodes, not {n:,}.")
203
+ spec.set_encoding(index, spec.STRING_ARRAY)
204
+ return obs
205
+
206
+
207
+ def write_var(root: Any, target: Target) -> None:
208
+ """var, with string columns encoded as anndata's writer would encode them.
209
+
210
+ anndata turns any string column with a repeated value into a categorical
211
+ when it writes (`strings_to_categoricals`), so an h5ad scanpy wrote from
212
+ the same file holds `feature_types` and `genome` as categoricals. Doing
213
+ the same keeps the two interchangeable, and those columns a few bytes
214
+ instead of a string per feature.
215
+ """
216
+ var = write_dataframe_header(root, "var", target.var_names, list(target.columns))
217
+ for key, values in target.columns.items():
218
+ if values.dtype.kind == "b":
219
+ write_dense(var, key, values)
220
+ continue
221
+ categories, codes = np.unique(values, return_inverse=True)
222
+ if len(categories) < len(values):
223
+ write_categorical(var, key, codes, list(categories))
224
+ else:
225
+ write_string_array(var, key, values)
226
+
227
+
228
+ # ---------------------------------------------------------------------------
229
+ # export
230
+
231
+
232
+ def matrix_path(layer: Optional[str]) -> str:
233
+ """`--layer` as a path: X, raw/X, a layer's name, or layers/NAME."""
234
+ if layer is None or layer in ("X", "raw/X") or layer.startswith("layers/"):
235
+ return layer or "X"
236
+ return f"layers/{layer}"
237
+
238
+
239
+ def resolve(root: Any, path: str) -> Any:
240
+ node = root
241
+ for part in path.split("/"):
242
+ if not is_group(node) or part not in node:
243
+ raise KeyError(f"{path!r} not found in the store.")
244
+ node = node[part]
245
+ return node
246
+
247
+
248
+ @dataclass
249
+ class ExportFeatures:
250
+ ids: List[str]
251
+ names: List[str]
252
+ types: List[str]
253
+ genomes: List[str]
254
+ tags: Dict[str, List[str]]
255
+
256
+
257
+ @dataclass
258
+ class MatrixSource:
259
+ """The matrix an export writes, and what is known about it before writing."""
260
+
261
+ path: str
262
+ n_obs: int
263
+ n_var: int
264
+ dtype: np.dtype
265
+ features: ExportFeatures
266
+
267
+
268
+ def plan_matrix_source(
269
+ source: Path, options: Options, warnings: List[str]
270
+ ) -> MatrixSource:
271
+ """Check the matrix `--layer` names against obs and var; read var's features.
272
+
273
+ Warns when the matrix holds floats, which 10x formats are not expected to.
274
+ """
275
+ path = matrix_path(options.layer)
276
+ with open_store(source, "r", require_anndata=False) as store:
277
+ root = store.root
278
+ matrix = describe(resolve(root, path))
279
+ var_path = "raw/var" if path.startswith("raw/") else "var"
280
+ var, obs = resolve(root, var_path), resolve(root, "obs")
281
+ obs_index, _ = resolve_index(obs, "obs")
282
+ var_index, _ = resolve_index(var, "var")
283
+ n_obs, n_var = element_len(obs_index), element_len(var_index)
284
+ if matrix.shape != (n_obs, n_var):
285
+ raise ValueError(
286
+ f"{path} is {matrix.shape[0]} x {matrix.shape[1]}, but obs has "
287
+ f"{n_obs} and {var_path} has {n_var} entries."
288
+ )
289
+ features = export_features(root, var, var_index, options, warnings)
290
+
291
+ if matrix.dtype.kind == "f":
292
+ warnings.append(
293
+ f"{path} holds {matrix.dtype} values. Cell Ranger writes integer "
294
+ "counts, and tools that read its output may expect them: export "
295
+ "a counts layer with --layer, or recast to int32 first."
296
+ )
297
+ return MatrixSource(
298
+ path=path, n_obs=n_obs, n_var=n_var, dtype=matrix.dtype, features=features
299
+ )
300
+
301
+
302
+ def export_features(
303
+ root: Any,
304
+ var: Any,
305
+ var_index: Any,
306
+ options: Options,
307
+ warnings: List[str],
308
+ ) -> ExportFeatures:
309
+ """Work out each feature's id, name, type and genome from var."""
310
+ columns = dataframe_columns(var)
311
+ n = element_len(var_index)
312
+
313
+ def column(name: str) -> List[str]:
314
+ return read_str_all(var[name], parent_group=var)
315
+
316
+ def requested(flag: str, name: Optional[str]) -> Optional[str]:
317
+ if name is not None and name not in columns:
318
+ raise ValueError(
319
+ f"{flag} {name}: no such var column. It has: "
320
+ f"{', '.join(columns) or 'none'}."
321
+ )
322
+ return name
323
+
324
+ ids_col = requested("--gene-ids-column", options.gene_ids_column)
325
+ names_col = requested("--gene-names-column", options.gene_names_column)
326
+ types_col = requested("--feature-types-column", options.feature_types_column)
327
+ index = read_str_all(var_index)
328
+
329
+ missing = []
330
+ if ids_col is None and names_col is None and "gene_ids" not in columns and "gene_symbols" in columns:
331
+ # var_names are ids already, as `--var-names gene_ids` leaves them.
332
+ ids, names = index, column("gene_symbols")
333
+ else:
334
+ names = column(names_col) if names_col else index
335
+ if ids_col is None and "gene_ids" in columns:
336
+ ids_col = "gene_ids"
337
+ if ids_col is not None:
338
+ ids = column(ids_col)
339
+ else:
340
+ missing.append("gene ids (a gene_ids column, or --gene-ids-column)")
341
+ ids = names
342
+ if types_col is None and "feature_types" in columns:
343
+ types_col = "feature_types"
344
+ if types_col is not None:
345
+ types = column(types_col)
346
+ else:
347
+ missing.append("feature types (a feature_types column, or --feature-types-column)")
348
+ types = ["Gene Expression"] * n
349
+
350
+ if missing:
351
+ if not options.force:
352
+ raise ValueError(
353
+ f"var has no {' and no '.join(missing)}, which a 10x matrix "
354
+ "records for every feature. Pass --force to write placeholders: "
355
+ "the names as ids, and 'Gene Expression' as the type."
356
+ )
357
+ warnings.append(f"Writing placeholders for {' and '.join(missing)}.")
358
+
359
+ genomes = column("genome") if "genome" in columns else [""] * n
360
+ tags: Dict[str, List[str]] = {}
361
+ for key in recorded_tag_keys(root):
362
+ if key != "genome" and key in columns:
363
+ tags[key] = column(key)
364
+ return ExportFeatures(ids=ids, names=names, types=types, genomes=genomes, tags=tags)
365
+
366
+
367
+ def cellranger_meta(root: Any) -> Optional[Any]:
368
+ """`uns/cellranger`, as an import recorded it, if there is one."""
369
+ if "uns" not in root or "cellranger" not in root["uns"]:
370
+ return None
371
+ meta = root["uns"]["cellranger"]
372
+ return meta if is_group(meta) else None
373
+
374
+
375
+ def recorded_tag_keys(root: Any) -> List[str]:
376
+ meta = cellranger_meta(root)
377
+ if meta is None or "feature_tag_keys" not in meta:
378
+ return []
379
+ return read_str_all(meta["feature_tag_keys"])
380
+