pyadata-cli 0.9.0__tar.gz → 0.10.0__tar.gz

{pyadata_cli-0.9.0 → pyadata_cli-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyadata-cli
-Version: 0.9.0
+Version: 0.10.0
 Summary: Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores
 Keywords: anndata,h5ad,zarr,bioinformatics,single-cell,cli
 Author: Aljes Binkevich
@@ -141,6 +141,7 @@ adata convert data.zarr -o data.h5ad --compression gzip
 adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
 adata convert counts.h5ad -o for_cellbender.h5 --layer counts
 adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
+adata convert splitpipe_out -o parse.h5ad --unfiltered
 ```
 
 ## Documentation

{pyadata_cli-0.9.0 → pyadata_cli-0.10.0}/README.md

@@ -101,6 +101,7 @@ adata convert data.zarr -o data.h5ad --compression gzip
 adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad --feature-type "Gene Expression"
 adata convert counts.h5ad -o for_cellbender.h5 --layer counts
 adata convert Solo.out/Velocyto/raw -o velocyto.h5ad
+adata convert splitpipe_out -o parse.h5ad --unfiltered
 ```
 
 ## Documentation

{pyadata_cli-0.9.0 → pyadata_cli-0.10.0}/pyproject.toml

@@ -3,7 +3,7 @@
 # so the distribution is published as `pyadata-cli`. The import package
 # and the command are both still `adata`.
 name = "pyadata-cli"
-version = "0.9.0"
+version = "0.10.0"
 description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores"
 readme = "README.md"
 requires-python = ">=3.12"

{pyadata_cli-0.9.0 → pyadata_cli-0.10.0}/src/adata/cli.py

@@ -524,6 +524,9 @@ _CONVERT_FLAGS = {
     "feature_types_column": "--feature-types-column",
     "raw": "--raw",
     "min_cell_probability": "--min-cell-probability",
+    "sample": "--sample",
+    "all_samples": "--all-samples",
+    "unfiltered": "--unfiltered",
 }
 
 #: Flags that only make sense for format conversion; with recast's flags,
@@ -608,9 +611,10 @@ def convert(
         "Cell Ranger 2 matrix)",
     ),
     var_names: str = typer.Option(
-        "gene_symbols",
+        "gene_ids",
         "--var-names",
-        help="10x input: name features by gene_symbols or gene_ids",
+        help="10x input: name features by gene_ids (default; unique) or "
+        "gene_symbols (scanpy's default)",
     ),
     make_unique: bool = typer.Option(
         False,
@@ -662,6 +666,21 @@ def convert(
         help="CellBender input: keep droplets with cell_probability at least "
         "this (CellBender's own cutoff is 0.5)",
     ),
+    sample: Optional[str] = typer.Option(
+        None,
+        "--sample",
+        help="Parse input: the sample folder to read (default: all-sample)",
+    ),
+    all_samples: bool = typer.Option(
+        False,
+        "--all-samples",
+        help="Parse input: write one output per sample, OUT.<sample>.h5ad",
+    ),
+    unfiltered: bool = typer.Option(
+        False,
+        "--unfiltered",
+        help="Parse input: read DGE_unfiltered (every barcode) not DGE_filtered",
+    ),
     # Deprecated: recast's flags, so `convert` keeps working as it did until
     # 1.0.0. Hidden from --help; the old matrix paths arrive in ctx.args.
     inplace: bool = typer.Option(False, "--inplace", hidden=True),
@@ -680,10 +699,13 @@ def convert(
     writes a Matrix Market folder. Between h5ad and zarr every element is
     copied, streamed, keeping the source's chunking and compression where the
     target can express them. CellBender output keeps its latent variables, as
-    CellBender's own loader names them. A 10x matrix is read as scanpy's read_10x_h5 or
-    read_10x_mtx (gex_only=False) reads it, keeping integer counts, and
-    written as Cell Ranger writes it. MTX folders from Cell Ranger and
-    STARsolo, velocyto's included, are read in two streaming passes.
+    CellBender's own loader names them. A 10x matrix is read as scanpy's
+    read_10x_h5 or read_10x_mtx (gex_only=False) reads it, keeping integer
+    counts, and written as Cell Ranger writes it. MTX folders from Cell
+    Ranger and STARsolo, velocyto's included, and Parse split-pipe output
+    are read in two streaming passes. Features are named by gene id, unique
+    but in probe-level (Flex) matrices; --var-names gene_symbols names them
+    as scanpy does.
 
     Nothing is written until every check has passed, and the output is moved
     into place only once complete: a failed conversion leaves nothing
@@ -704,6 +726,8 @@ def convert(
         adata convert counts.h5ad -o counts_mtx --to 10x-mtx --layer counts
         adata convert cellbender_output.h5 -o cb.h5ad --raw raw_feature_bc_matrix.h5
         adata convert cellbender_output.h5 -o cells.h5ad --min-cell-probability 0.5
+        adata convert splitpipe_out -o parse.h5ad
+        adata convert splitpipe_out -o parse.h5ad --all-samples --unfiltered
     """
     entries = list(ctx.args)
     recast_flags = _given(ctx, _RECAST_FLAGS)
@@ -789,6 +813,9 @@ def convert(
                 feature_types_column=feature_types_column,
                 raw=raw,
                 min_cell_probability=min_cell_probability,
+                sample=sample,
+                all_samples=all_samples,
+                unfiltered=unfiltered,
             ),
         )
     except Exception as e:

{pyadata_cli-0.9.0 → pyadata_cli-0.10.0}/src/adata/conversion/__init__.py

@@ -17,7 +17,7 @@ from typing import Dict, FrozenSet, List, Optional
 
 import h5py
 
-from adata.conversion import cellbender, mtx, tenx
+from adata.conversion import cellbender, mtx, parse, tenx
 from adata.conversion.anndata_copy import AnnDataCopy
 from adata.conversion.base import ConversionPlan, Converter, Format, Options
 from adata.storage import has_valid_anndata_root_attrs, is_zarr_path
@@ -92,6 +92,14 @@ FORMATS: Dict[str, Format] = {
         importer=mtx.MtxImport(),
         exporter=mtx.MtxExport(),
     ),
+    # A folder too; read only.
+    "parse": Format(
+        name="parse",
+        description="Parse Biosciences split-pipe output (a run, sample or DGE folder)",
+        extensions=(),
+        sniff=parse.sniff,
+        importer=parse.ParseImport(),
+    ),
 }
 
 _ANNDATA_COPY = AnnDataCopy()

{pyadata_cli-0.9.0 → pyadata_cli-0.10.0}/src/adata/conversion/base.py

@@ -24,7 +24,7 @@ class Options:
 
     # Reading a 10x matrix.
     genome: Optional[str] = None
-    var_names: str = "gene_symbols"
+    var_names: str = "gene_ids"
     make_unique: bool = False
     feature_type: Optional[str] = None
     split_feature_types: bool = False
@@ -39,6 +39,11 @@ class Options:
     raw: Optional[Path] = None
     min_cell_probability: Optional[float] = None
 
+    # Reading Parse split-pipe output.
+    sample: Optional[str] = None
+    all_samples: bool = False
+    unfiltered: bool = False
+
 
 @dataclass
 class ConversionPlan:

{pyadata_cli-0.9.0 → pyadata_cli-0.10.0}/src/adata/conversion/features.py

@@ -40,7 +40,12 @@ from adata.elements.write import (
     write_string_array,
 )
 from adata.storage import create_dataset, is_group, is_zarr_group, open_store
-from adata.util.names import count_duplicates, make_index_unique, unique_names
+from adata.util.names import (
+    count_duplicates,
+    make_index_unique,
+    natural_sorted,
+    unique_names,
+)
 
 VAR_NAMES = ("gene_symbols", "gene_ids")
 
@@ -218,9 +223,10 @@ def write_var(root: Any, target: Target) -> None:
         if values.dtype.kind in "biuf":
             write_dense(var, key, values)
             continue
-        categories, codes = np.unique(values, return_inverse=True)
+        categories = natural_sorted(set(values.tolist()))
         if len(categories) < len(values):
-            write_categorical(var, key, codes, list(categories))
+            lookup = {c: i for i, c in enumerate(categories)}
+            write_categorical(var, key, [lookup[v] for v in values.tolist()], categories)
         else:
             write_string_array(var, key, values)
 

pyadata_cli-0.10.0/src/adata/conversion/parse.py

@@ -0,0 +1,404 @@
+"""Parse Biosciences split-pipe output.
+
+split-pipe writes, for the combined run and for each sample it was told of:
+
+    <run>/all-sample/DGE_filtered/{count_matrix.mtx, all_genes.csv, cell_metadata.csv}
+    <run>/all-sample/DGE_unfiltered/   (every barcode, not only cells)
+    <run>/<sample>/DGE_filtered/ ...
+
+(`all-well` rather than `all-sample` in older versions.) The matrix is
+cells x genes, the other way round from 10x, so each file row is a row of X.
+`all_genes.csv` holds `gene_id,gene_name,genome`, and `cell_metadata.csv`
+starts with `bc_wells`, the barcode, followed by the sample, the species and
+per-cell counts.
+
+The result matches Parse's scanpy recipe -- `read_mtx` plus `pd.read_csv`
+of both tables, indexed by `bc_wells` and `gene_name` -- except that
+features are named by gene id unless `--var-names gene_symbols` is given,
+and integer counts stay integers.
+"""
+
+from __future__ import annotations
+
+from contextlib import ExitStack
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Sequence
+
+import numpy as np
+from rich.console import Console
+
+from adata.conversion.base import ConversionPlan, Converter, Options
+from adata.conversion.features import (
+    Target,
+    check_import_options,
+    plan_targets,
+    write_var,
+)
+from adata.core.csr_build import CsrAppender
+from adata.elements.write import ensure_anndata_skeleton
+from adata.formats.mtx import EntryMap, MtxHeader, read_header, read_into, scan, value_dtype
+from adata.formats.table import TableScan, read_small_table, scan_table, write_table
+from adata.storage import COMPRESSIONS, open_store
+from adata.util.names import unique_names
+from adata.util.path import is_zarr_path
+
+_FILTERED, _UNFILTERED = "DGE_filtered", "DGE_unfiltered"
+
+#: The combined folder, by split-pipe version.
+_COMBINED = ("all-sample", "all-well")
+
+_INDEX = "bc_wells"
+
+
+# ---------------------------------------------------------------------------
+# finding the files
+
+
+@dataclass
+class _Dge:
+    path: Path
+    matrix: Path
+    genes: Path
+    cells: Path
+
+    @property
+    def sample(self) -> str:
+        return self.path.parent.name
+
+    @property
+    def filtered(self) -> bool:
+        return self.path.name != _UNFILTERED
+
+
+def _dge(folder: Path) -> Optional[_Dge]:
+    """The DGE folder `folder` is, or None."""
+    if not folder.is_dir() or is_zarr_path(folder):
+        return None
+    matrix = next(
+        (p for p in (folder / "count_matrix.mtx", folder / "count_matrix.mtx.gz") if p.is_file()),
+        None,
+    )
+    genes, cells = folder / "all_genes.csv", folder / "cell_metadata.csv"
+    if matrix is None or not genes.is_file() or not cells.is_file():
+        return None
+    return _Dge(folder, matrix, genes, cells)
+
+
+def _sample(folder: Path, which: str) -> Optional[_Dge]:
+    """A sample folder's DGE folder of kind `which`, or None."""
+    return _dge(folder / which) if folder.is_dir() else None
+
+
+def _is_sample(folder: Path) -> bool:
+    return any(_sample(folder, which) is not None for which in (_FILTERED, _UNFILTERED))
+
+
+def _children(path: Path) -> List[Path]:
+    return sorted(p for p in path.iterdir() if p.is_dir()) if path.is_dir() else []
+
+
+def sniff(path: Path) -> bool:
+    path = Path(path)
+    if not path.is_dir() or is_zarr_path(path):
+        return False
+    # A folder of several runs is recognised too, so `locate` can say to
+    # convert them one at a time rather than "unknown format".
+    return (
+        _dge(path) is not None
+        or _is_sample(path)
+        or any(
+            _is_sample(child) or any(_is_sample(g) for g in _children(child))
+            for child in _children(path)
+        )
+    )
+
+
+def locate(path: Path, options: Options) -> List[_Dge]:
+    """The DGE folders to convert: one, or one per sample."""
+    path = Path(path)
+    which = _UNFILTERED if options.unfiltered else _FILTERED
+    picking = options.sample is not None or options.all_samples
+
+    found = _dge(path)
+    if found is not None:
+        if options.unfiltered or picking:
+            flag = "--unfiltered" if options.unfiltered else (
+                "--sample" if options.sample is not None else "--all-samples"
+            )
+            raise ValueError(
+                f"{path.name} is a DGE folder already, so {flag} has nothing to "
+                "choose from; give the split-pipe output folder instead."
+            )
+        return [found]
+
+    if _is_sample(path):
+        if picking:
+            raise ValueError(
+                f"{path.name} is a single sample's folder; give the split-pipe "
+                "output folder to choose samples."
+            )
+        return [_need(path, which)]
+
+    samples = {child.name: child for child in _children(path) if _is_sample(child)}
+    if not samples:
+        runs = [child.name for child in _children(path) if any(
+            _is_sample(grandchild) for grandchild in _children(child)
+        )]
+        if runs:
+            raise ValueError(
+                f"{path} holds {len(runs)} split-pipe runs ({', '.join(runs[:5])}"
+                f"{', ...' if len(runs) > 5 else ''}); convert one at a time."
+            )
+        raise ValueError(
+            f"{path} is not split-pipe output: it needs count_matrix.mtx, "
+            "all_genes.csv and cell_metadata.csv, in DGE_filtered/ or "
+            "DGE_unfiltered/ under each sample."
+        )
+
+    named = [name for name in samples if name not in _COMBINED]
+    if options.all_samples:
+        if not named:
+            raise ValueError(
+                f"{path.name} holds only the combined {', '.join(samples)} folder; "
+                "split-pipe was run without samples."
+            )
+        return [_need(samples[name], which) for name in named]
+    if options.sample is not None:
+        if options.sample not in samples:
+            raise ValueError(
+                f"No sample {options.sample!r}; this run has: {', '.join(samples)}."
+            )
+        return [_need(samples[options.sample], which)]
+    combined = next((name for name in _COMBINED if name in samples), None)
+    if combined is None and len(samples) == 1:
+        combined = next(iter(samples))
+    if combined is None:
+        raise ValueError(
+            f"{path.name} holds {len(samples)} samples ({', '.join(samples)}) and "
+            "no combined all-sample folder; pick one with --sample, or write "
+            "each with --all-samples."
+        )
+    return [_need(samples[combined], which)]
+
+
+def _need(sample: Path, which: str) -> _Dge:
+    found = _sample(sample, which)
+    if found is None:
+        other = _UNFILTERED if which == _FILTERED else _FILTERED
+        hint = (
+            f"; it has {other}/" + (" (pass --unfiltered)" if other == _UNFILTERED else "")
+            if _sample(sample, other) is not None else ""
+        )
+        raise ValueError(f"{sample.name} has no readable {which}/{hint}.")
+    return found
+
+
+# ---------------------------------------------------------------------------
+# import
+
+
+@dataclass
+class _Item:
+    dge: _Dge
+    header: MtxHeader
+    cells: TableScan
+    rows: Optional[np.ndarray]
+    target: Target
+    emap: EntryMap
+
+
+@dataclass
+class _Import:
+    items: List[_Item]
+    compression: Optional[str]
+    zarr_format: Optional[int]
+
+
+class ParseImport(Converter):
+    options = frozenset({
+        "--genome", "--var-names", "--make-unique", "--compression",
+        "--sample", "--all-samples", "--unfiltered",
+    })
+
+    def plan(
+        self,
+        source: Path,
+        output: Path,
+        src_format: str,
+        dst_format: str,
+        options: Options,
+    ) -> ConversionPlan:
+        check_import_options(options)
+        if options.sample is not None and options.all_samples:
+            raise ValueError(
+                "--sample picks one sample and --all-samples writes every one; "
+                "pass one or the other."
+            )
+        backend = "zarr" if dst_format == "zarr" else "hdf5"
+        if options.compression is not None and options.compression not in COMPRESSIONS[backend]:
+            raise ValueError(
+                f"--compression {options.compression} is not available for "
+                f"{dst_format}; use one of: {', '.join(COMPRESSIONS[backend])}."
+            )
+
+        dges = locate(Path(source), options)
+        stems = unique_names([dge.sample for dge in dges])
+        items: List[_Item] = []
+        outputs: List[Path] = []
+        warnings: List[str] = []
+        for dge in dges:
+            path = output
+            if options.all_samples:
+                path = output.with_name(f"{output.stem}.{stems[dge.sample]}{output.suffix}")
+            item, notes = _plan_item(dge, path, options)
+            items.append(item)
+            outputs.append(path)
+            warnings += [f"{dge.sample}: {n}" if options.all_samples else n for n in notes]
+
+        first = items[0]
+        n_cells = first.cells.n_rows if first.rows is None else first.rows.size
+        kept = first.target.keep
+        described = (
+            f"{n_cells:,} cells x "
+            f"{first.header.n_cols if kept is None else kept.size:,} genes"
+        )
+        if options.genome is not None:
+            described += f" of {options.genome}"
+        if len(items) > 1:
+            described = f"{len(items)} samples, one output each ({', '.join(d.sample for d in dges)})"
+        else:
+            described += f", {first.header.nnz:,} entries"
+        kind = "filtered" if first.dge.filtered else "unfiltered"
+        where = "" if len(items) > 1 else f"{first.dge.sample}, "
+        return ConversionPlan(
+            source=source,
+            src_format=src_format,
+            dst_format=dst_format,
+            outputs=outputs,
+            summary=f"Parse split-pipe ({where}{kind}) → {dst_format}, {described}",
+            warnings=warnings,
+            detail=_Import(
+                items=items,
+                compression=options.compression or ("gzip" if backend == "hdf5" else None),
+                zarr_format=(options.zarr_format or 3) if dst_format == "zarr" else None,
+            ),
+        )
+
+    def write(
+        self, plan: ConversionPlan, staged: Sequence[Path], console: Console
+    ) -> None:
+        detail: _Import = plan.detail
+
+        # Pass 1 of every matrix, before any output exists: a file that
+        # cannot be read must not leave a store holding obs and var alone.
+        scans = []
+        for item in detail.items:
+            with console.status(f"[cyan]Checking {_label(item)}...[/]"):
+                scans.append(scan(item.header, item.emap))
+
+        for item, scanned, path in zip(detail.items, scans, staged):
+            with ExitStack() as stack:
+                root = stack.enter_context(
+                    open_store(path, "w", zarr_format=detail.zarr_format)
+                ).root
+                with console.status(f"[cyan]Writing obs and var of {item.dge.sample}...[/]"):
+                    write_table(root, "obs", item.cells, rows=item.rows)
+                    write_var(root, item.target)
+                    ensure_anndata_skeleton(root)
+                appender = CsrAppender(
+                    root, "X",
+                    shape=(item.emap.n_rows, item.emap.target_cols[0]),
+                    data_dtype=value_dtype(item.header, scanned),
+                    compression=detail.compression,
+                )
+                with console.status(f"[cyan]Reading {_label(item)}...[/]"):
+                    summed = read_into(item.header, item.emap, scanned, [[appender]], root)
+                    appender.finish()
+                if summed:
+                    console.print(
+                        f"[yellow]Warning:[/] {summed:,} duplicate entries in "
+                        f"{_label(item)} were summed, as scipy does."
+                    )
+
+
+def _label(item: _Item) -> str:
+    return f"{item.dge.sample}/{item.dge.path.name}/{item.dge.matrix.name}"
+
+
+def _plan_item(dge: _Dge, output: Path, options: Options):
+    """Everything about one DGE folder, checked before any output exists."""
+    header = read_header(dge.matrix)
+    if header.field not in ("integer", "real", "pattern"):
+        raise ValueError(f"{dge.matrix.name} holds {header.field} values, which AnnData cannot.")
+
+    columns, rows = read_small_table(dge.genes)
+    for name in ("gene_id", "gene_name"):
+        if name not in columns:
+            raise ValueError(
+                f"{dge.genes.name} has no {name!r} column; it has: {', '.join(columns)}."
+            )
+    if len(rows) != header.n_cols:
+        raise ValueError(
+            f"{dge.genes.name} lists {len(rows):,} genes, but {dge.matrix.name} "
+            f"has {header.n_cols:,} columns."
+        )
+    field = {name: np.array([row[i] for row in rows], dtype=object) for i, name in enumerate(columns)}
+    names = field["gene_name"]
+    var = {"gene_ids": field["gene_id"]}
+    var.update({k: v for k, v in field.items() if k not in ("gene_id", "gene_name")})
+
+    select = None
+    keep = np.ones(header.n_cols, dtype=bool)
+    if options.genome is not None:
+        genomes = field.get("genome")
+        if genomes is None:
+            raise ValueError(f"{dge.genes.name} records no genome per gene to filter on.")
+        keep = genomes == options.genome
+        if not keep.any():
+            available = list(dict.fromkeys(genomes.tolist()))
+            raise ValueError(
+                f"No genes of genome {options.genome!r}; this run has: {', '.join(available)}."
+            )
+        select = ("species", options.genome)
+
+    cells = scan_table(dge.cells, index=_INDEX, select=select)
+    if cells.n_rows != header.n_rows:
+        raise ValueError(
+            f"{dge.cells.name} lists {cells.n_rows:,} cells, but {dge.matrix.name} "
+            f"has {header.n_rows:,} rows."
+        )
+    picked = None
+    if select is not None:
+        picked = cells.selected
+        if picked.size == 0:
+            raise ValueError(
+                f"No cell in {dge.cells.name} has species {options.genome!r}."
+            )
+
+    _, (target,), notes = plan_targets(
+        output, names, var, None, keep, options, noun="run"
+    )
+    row_map = None
+    if picked is not None:
+        row_map = np.full(header.n_rows, -1, dtype=np.int64)
+        row_map[picked] = np.arange(picked.size)
+    emap = _entry_map(
+        picked.size if picked is not None else header.n_rows, header.n_cols, target, row_map
+    )
+    return _Item(dge, header, cells, picked, target, emap), notes
+
+
+def _entry_map(
+    n_rows: int, n_genes: int, target: Target, row_map: Optional[np.ndarray]
+) -> EntryMap:
+    if target.keep is None:
+        return EntryMap(n_rows=n_rows, n_cols=n_genes, row_map=row_map)
+    owner = np.full(n_genes, -1, dtype=np.int64)
+    local = np.zeros(n_genes, dtype=np.int64)
+    owner[target.keep] = 0
+    local[target.keep] = np.arange(target.keep.size)
+    return EntryMap(
+        n_rows=n_rows, n_cols=n_genes, owner=owner, local=local,
+        target_cols=[int(target.keep.size)], row_map=row_map,
+    )

{pyadata_cli-0.9.0 → pyadata_cli-0.10.0}/src/adata/elements/write.py

@@ -127,6 +127,22 @@ def write_categorical(
     return group
 
 
+def create_categorical(
+    parent: Any, name: str, categories: Sequence[Any], n: int, ordered: bool = False
+) -> Any:
+    """A `categorical` of `n` values whose codes the caller fills in blocks.
+
+    Returns the `codes` array, sized and typed for `categories`.
+    """
+    group = parent.create_group(name)
+    spec.set_encoding(group, spec.CATEGORICAL)
+    group.attrs["ordered"] = bool(ordered)
+    write_string_array(group, "categories", np.asarray(list(categories), dtype=object))
+    codes = create_dataset(group, "codes", shape=(n,), dtype=_codes_dtype(len(categories)))
+    spec.set_encoding(codes, spec.ARRAY)
+    return codes
+
+
 def _codes_dtype(n_categories: int) -> Any:
     """Smallest signed integer dtype that can index `n_categories` plus -1."""
     if n_categories < 128:

{pyadata_cli-0.9.0 → pyadata_cli-0.10.0}/src/adata/formats/mtx.py

@@ -216,7 +216,9 @@ class EntryMap:
     feature (the output's column axis), split the features among several
     outputs: `owner` gives each feature's output, or -1 to drop it, and
     `local` its column there. Outputs are numbered within one key space,
-    `key = output * n_rows + row`, so one sort orders them all.
+    `key = output * n_rows + row`, so one sort orders them all. `row_map`,
+    indexed by the file's row (after `transpose`), gives each its output row,
+    or -1 to drop it; `n_rows` then counts output rows.
     """
 
     n_rows: int
@@ -226,6 +228,7 @@ class EntryMap:
     local: Optional[np.ndarray] = None
     n_targets: int = 1
     target_cols: List[int] = field(default_factory=list)
+    row_map: Optional[np.ndarray] = None
 
     def __post_init__(self) -> None:
         if not self.target_cols:
@@ -235,6 +238,10 @@ class EntryMap:
         self, rows: np.ndarray, cols: np.ndarray, values: np.ndarray
     ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
         out_rows, features = (cols, rows) if self.transpose else (rows, cols)
+        if self.row_map is not None:
+            out_rows = self.row_map[out_rows]
+            kept = out_rows >= 0
+            out_rows, features, values = out_rows[kept], features[kept], values[kept]
         if self.owner is None:
             return out_rows, features, values
         owner = self.owner[features]

pyadata_cli-0.10.0/src/adata/formats/table.py

@@ -0,0 +1,364 @@
+"""Delimited text tables, read in two streaming passes.
+
+Per-cell metadata arrives as CSV (Parse's `cell_metadata.csv`) with a row
+per cell, so a mega run's table has a million rows. `pd.read_csv` would hold
+it whole; here the file is read a block of lines at a time, twice.
+
+The first pass finds each column's type as `pd.read_csv` would: `int64`,
+`float64` when a number is missing or fractional, `bool` for a column of
+True/False, and text otherwise, with pandas' own list of strings that mean
+"missing". The second writes the columns as anndata would write that pandas
+frame: a text column becomes a categorical when it repeats a value
+(anndata's `strings_to_categoricals`), and a string array when every value
+is distinct.
+
+Two limits, both deliberate:
+
+* A text column with more than `CATEGORY_CAP` distinct values is written as
+  strings, even when some repeat, since tracking its values would hold the
+  column in memory.
+* A quoted field may not contain a newline: the file is cut into blocks at
+  line ends. Such a row is refused with its line number.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional, Set, Tuple
+
+import numpy as np
+
+from adata.elements import spec
+from adata.elements.strings import string_dtype_for
+from adata.elements.write import create_categorical, set_column_order, write_string_array
+from adata.formats.textio import iter_line_blocks, open_binary
+from adata.storage import create_dataset, is_zarr_group
+from adata.util.names import natural_sorted
+
+#: Distinct values tracked per text column before it is left as strings.
+CATEGORY_CAP = 1 << 16
+
+#: Bytes of a table parsed at once. Smaller than for MTX, because a parsed
+#: row is a list of Python strings, several times the size of its text.
+TABLE_BLOCK_BYTES = 1 << 20
+
+#: What `pd.read_csv` reads as missing, by default.
+NA_VALUES = frozenset({
+    "", "#N/A", "#N/A N/A", "#NA", "-1.#IND", "-1.#QNAN", "-NaN", "-nan",
+    "1.#IND", "1.#QNAN", "<NA>", "N/A", "NA", "NULL", "NaN", "None", "n/a",
+    "nan", "null",
+})
+
+_TRUE = frozenset({"True", "TRUE", "true"})
+_FALSE = frozenset({"False", "FALSE", "false"})
+
+# A column's kind narrows as values are seen: bool -> int -> float -> text.
+_ORDER = ("bool", "int", "float", "text")
+
+
+@dataclass
+class Column:
+    name: str
+    kind: str = "bool"
+    has_missing: bool = False
+    #: Distinct values of a text column, None once past the cap.
+    seen: Optional[Set[str]] = field(default_factory=set)
+
+    @property
+    def categories(self) -> Optional[List[str]]:
+        """The categories, in the order anndata gives them."""
+        return None if self.seen is None else natural_sorted(self.seen)
+
+
+@dataclass
+class TableScan:
+    path: Path
+    delimiter: str
+    header: List[str]
+    index: str
+    n_rows: int
+    columns: List[Column]
+    #: Rows (0-based, after the header) where `select` matched, if asked.
+    selected: Optional[np.ndarray] = None
+
+    def dtype_of(self, column: Column) -> str:
+        """What the column is written as: int64, float64, bool, category or str."""
+        if column.kind == "int":
+            return "float64" if column.has_missing else "int64"
+        if column.kind == "float":
+            return "float64"
+        if column.kind == "bool":
+            if not column.seen:
+                return "float64"  # nothing but missing values, as pandas reads it
+            # pandas reads True/False with gaps as text.
+            return "category" if column.has_missing else "bool"
+        return "category" if self.is_categorical(column) else "str"
+
+    def is_categorical(self, column: Column) -> bool:
+        if column.kind not in ("text", "bool") or column.seen is None:
+            return False
+        # anndata's rule: fewer categories (missing values are none) than
+        # rows. So a column of distinct values and one gap is categorical.
+        return len(column.seen) < self.n_rows
+
+
+def _rows(
+    path: Path, delimiter: str, block_bytes: int
+) -> Iterator[Tuple[int, List[List[str]]]]:
+    """Blocks of parsed rows, each with the file line number of its first."""
+    line = 1
+    with open_binary(path) as fh:
+        for block in iter_line_blocks(fh, block_bytes):
+            reader = csv.reader(io.StringIO(block.decode("utf-8")), delimiter=delimiter, strict=True)
+            rows: List[List[str]] = []
+            start = line
+            try:
+                for row in reader:
+                    rows.append(row)
+            except csv.Error as exc:
+                raise ValueError(
+                    f"{path.name}, line {start + reader.line_num - 1}: {exc}. "
+                    "A quoted field may not span lines."
+                ) from exc
+            line += block.count(b"\n")
+            yield start, rows
+
+
+def _narrow(column: Column, values: List[str]) -> None:
+    """Narrow `column`'s kind to cover `values`."""
+    present = [v for v in values if v not in NA_VALUES]
+    if len(present) < len(values):
+        column.has_missing = True
+    kind = _ORDER.index(column.kind)
+    while present and kind < 3:
+        try:
+            if kind == 0:
+                if not all(v in _TRUE or v in _FALSE for v in present):
+                    raise ValueError
+            elif kind == 1:
+                np.array(present, dtype=np.int64)
+            else:
+                np.array(present, dtype=np.float64)
+            break
+        except (ValueError, OverflowError):
+            kind += 1
+    column.kind = _ORDER[kind]
+    if column.seen is not None:
+        column.seen.update(present)
+        if len(column.seen) > CATEGORY_CAP:
+            column.seen = None
+
+
+def scan_table(
+    path: Path,
+    *,
+    index: Optional[str] = None,
+    delimiter: str = ",",
+    select: Optional[Tuple[str, str]] = None,
+    block_bytes: int = TABLE_BLOCK_BYTES,
+) -> TableScan:
+    """Pass 1: the header, the row count and each column's type.
+
+    `index` names the column that becomes the row names (the first, if not
+    given). `select=(column, value)` also records the rows where that column
+    holds that value.
+    """
+    path = Path(path)
+    header: Optional[List[str]] = None
+    columns: List[Column] = []
+    positions: List[int] = []
+    pick = -1
+    selected: List[np.ndarray] = []
+    n_rows = 0
+    for start, rows in _rows(path, delimiter, block_bytes):
+        if header is None:
+            rows = [row for row in rows if row]
+            if not rows:
+                continue
+            header, rows = rows[0], rows[1:]
+            if len(set(header)) != len(header):
+                repeated = sorted({h for h in header if header.count(h) > 1})
+                raise ValueError(f"{path.name} repeats the column {', '.join(map(repr, repeated))}.")
+            index = header[0] if index is None else index
+            if index not in header:
+                raise ValueError(f"{path.name} has no {index!r} column; it has: {', '.join(header)}.")
+            positions = [i for i, name in enumerate(header) if name != index]
+            columns = [Column(header[i]) for i in positions]
+            if select is not None:
+                if select[0] not in header:
+                    raise ValueError(f"{path.name} has no {select[0]!r} column.")
+                pick = header.index(select[0])
+            start += 1
+        _check_widths(path, rows, len(header), start)
+        rows = [row for row in rows if row]
+        if not rows:
+            continue
+        by_field = list(zip(*rows))
+        for column, i in zip(columns, positions):
+            _narrow(column, list(by_field[i]))
+        if pick >= 0:
+            hits = np.flatnonzero(np.asarray(by_field[pick], dtype=object) == select[1])
+            selected.append(hits + n_rows)
+        n_rows += len(rows)
+    if header is None:
+        raise ValueError(f"{path.name} is empty: it has no header line.")
+    return TableScan(
+        path=path,
+        delimiter=delimiter,
+        header=header,
+        index=index,
+        n_rows=n_rows,
+        columns=columns,
+        selected=(
+            np.concatenate(selected) if selected else np.zeros(0, dtype=np.int64)
+        ) if select is not None else None,
+    )
+
+
+def _check_widths(path: Path, rows: List[List[str]], width: int, start: int) -> None:
+    for offset, row in enumerate(rows):
+        if row and len(row) != width:
+            raise ValueError(
+                f"{path.name}, line {start + offset}: {len(row)} fields where "
+                f"the header has {width}."
+            )
+
+
+def iter_table_blocks(
+    scan: TableScan,
+    *,
+    rows: Optional[np.ndarray] = None,
+    block_bytes: int = TABLE_BLOCK_BYTES,
+) -> Iterator[Tuple[List[str], Dict[str, List[str]]]]:
+    """Pass 2: (index values, {column: values}) a block of rows at a time.
+
+    `rows`, ascending, keeps only those rows.
+    """
+    at = scan.header.index(scan.index)
+    done = 0
+    first = True
+    for _, block in _rows(scan.path, scan.delimiter, block_bytes):
+        block = [row for row in block if row]
+        if first and block:
+            block, first = block[1:], False
+        if not block:
+            continue
+        if rows is not None:
+            lo, hi = np.searchsorted(rows, (done, done + len(block)))
+            picked = [block[i] for i in rows[lo:hi] - done]
+            done += len(block)
+            block = picked
+            if not block:
+                continue
+        else:
+            done += len(block)
+        by_field = list(zip(*block))
+        yield list(by_field[at]), {
+            name: list(by_field[i]) for i, name in enumerate(scan.header) if name != scan.index
+        }
+
+
+def _values(scan: TableScan, column: Column, values: List[str], lookup: Dict[str, int]) -> np.ndarray:
+    kind = scan.dtype_of(column)
+    if kind == "int64":
+        return np.array(values, dtype=np.int64)
+    if kind == "float64":
+        return np.array([np.nan if v in NA_VALUES else v for v in values], dtype=np.float64)
+    if kind == "bool":
+        return np.array([v in _TRUE for v in values], dtype=bool)
+    if kind == "category":
+        return np.array([lookup.get(v, -1) for v in values], dtype=np.int64)
+    return np.array(["" if v in NA_VALUES else v for v in values], dtype=object)
+
+
+def write_table(
+    parent: Any,
+    name: str,
+    scan: TableScan,
+    *,
+    rows: Optional[np.ndarray] = None,
+    block_bytes: int = TABLE_BLOCK_BYTES,
+) -> Any:
+    """Pass 2: the table as a `dataframe` named `name` under `parent`.
+
+    Every array is created at its full length first and filled a block of
+    rows at a time, so memory is set by the block, not by the table.
+    """
+    n = scan.n_rows if rows is None else int(rows.size)
+    backend = "zarr" if is_zarr_group(parent) else "hdf5"
+    group = parent.create_group(name)
+    spec.set_encoding(group, spec.DATAFRAME)
+    group.attrs["_index"] = "_index"
+    set_column_order(group, [c.name for c in scan.columns])
+
+    index = create_dataset(group, "_index", shape=(n,), dtype=string_dtype_for(backend))
+    spec.set_encoding(index, spec.STRING_ARRAY)
+    targets: Dict[str, Any] = {}
+    lookups: Dict[str, Dict[str, int]] = {}
+    for column in scan.columns:
+        kind = scan.dtype_of(column)
+        if kind == "category":
+            categories = column.categories or []
+            targets[column.name] = create_categorical(group, column.name, categories, n)
+            lookups[column.name] = {c: i for i, c in enumerate(categories)}
+            continue
+        dtype = string_dtype_for(backend) if kind == "str" else np.dtype(kind)
+        targets[column.name] = create_dataset(group, column.name, shape=(n,), dtype=dtype)
+        spec.set_encoding(
+            targets[column.name], spec.STRING_ARRAY if kind == "str" else spec.ARRAY
+        )
+
+    used = {name: np.zeros(len(lookup), dtype=bool) for name, lookup in lookups.items()}
+    written = 0
+    for names, block in iter_table_blocks(scan, rows=rows, block_bytes=block_bytes):
+        end = written + len(names)
+        index[written:end] = np.asarray(names, dtype=object)
+        for column in scan.columns:
+            values = _values(scan, column, block[column.name], lookups.get(column.name, {}))
+            targets[column.name][written:end] = values
+            if column.name in used:
+                used[column.name][values[values >= 0]] = True
+        written = end
+    if written != n:
+        raise ValueError(f"{scan.path.name} changed while it was read: {written:,} rows, not {n:,}.")
+    if rows is not None:
+        for name, seen in used.items():
+            if not seen.all():
+                _drop_unused(group, name, seen, n, block_rows=max(1, block_bytes // 8))
+    return group
+
+
+def _drop_unused(group: Any, name: str, used: np.ndarray, n: int, block_rows: int) -> None:
+    """Remove categories no written row uses, as anndata does on subsetting."""
+    remap = np.full(used.size + 1, -1, dtype=np.int64)  # the last slot maps -1
+    remap[:-1][used] = np.arange(int(used.sum()))
+    categorical = group[name]
+    categories = np.asarray(categorical["categories"][...], dtype=object)[used]
+    del categorical["categories"]
+    write_string_array(categorical, "categories", categories)
+    codes = categorical["codes"]
+    for lo in range(0, n, block_rows):
+        block = np.asarray(codes[lo : lo + block_rows], dtype=np.int64)
+        codes[lo : lo + block.size] = remap[block]
+
+
+def read_small_table(path: Path, delimiter: str = ",") -> Tuple[List[str], List[List[str]]]:
+    """The header and every row of a table small enough to hold, such as a
+    gene list."""
+    header: Optional[List[str]] = None
+    body: List[List[str]] = []
+    for start, rows in _rows(Path(path), delimiter, TABLE_BLOCK_BYTES):
+        if header is None:
+            rows = [row for row in rows if row]
+            if not rows:
+                continue
+            header, rows = rows[0], rows[1:]
+            start += 1
+        _check_widths(Path(path), rows, len(header), start)
+        body.extend(row for row in rows if row)
+    if header is None:
+        raise ValueError(f"{Path(path).name} is empty: it has no header line.")
+    return header, body

{pyadata_cli-0.9.0 → pyadata_cli-0.10.0}/src/adata/util/names.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import re
 from collections import Counter
-from typing import Dict, List, Sequence
+from typing import Dict, Iterable, List, Sequence
 
 
 def sanitize_label(label: str) -> str:
@@ -67,3 +67,20 @@ def make_index_unique(names: Sequence[str], join: str = "-") -> List[str]:
                 values[i] = candidate
                 break
     return values
+
+
+_DIGITS = re.compile(r"(\d+)")
+
+
+def natural_key(text: str) -> tuple:
+    """Sort key putting "A2" before "A10", as natsort's default does."""
+    parts = _DIGITS.split(text)
+    return tuple(int(part) if i % 2 else part for i, part in enumerate(parts))
+
+
+def natural_sorted(values: Iterable[str]) -> List[str]:
+    """`values` in natsort's default order, which is how anndata orders the
+    categories it makes from strings (`strings_to_categoricals`). Equal keys
+    ("01" and "1") keep their plain sorted order, as natsort's stable sort
+    of pandas' sorted categories does."""
+    return sorted(sorted(values), key=natural_key)