pyadata-cli 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

adata/cli.py

@@ -6,11 +6,13 @@ from typing import Optional, Sequence, List
 from rich.console import Console
 import typer
 
-from adata.core.convert import LAYOUTS
+from adata.conversion import FORMATS, Options
+from adata.core.recast import LAYOUTS
 from adata.commands import (
     MERGE_CHOICES,
     concat_stores,
     convert_store,
+    recast_store,
     create_store,
     split_store,
     list_store,
@@ -26,7 +28,7 @@ from adata.commands import export_image as export_image_cmd
 
 app = typer.Typer(
     help="Streaming CLI for huge AnnData .h5ad and .zarr stores "
-    "(view, ls, subset, split, concat, export, import)."
+    "(view, ls, create, subset, split, concat, convert, recast, export, import)."
 )
 # Use stderr for status/progress to keep stdout clean for data output
 # force_terminal=True ensures Rich output is visible even in non-TTY environments
@@ -481,8 +483,285 @@ def concat(
 # ============================================================================
 # CONVERT command
 # ============================================================================
-@app.command("convert")
+#: The flags `recast` has and format conversion does not. Until 1.0.0, any of
+#: them on `convert` means the old meaning of the command, and it forwards.
+_RECAST_FLAGS = {
+    "inplace": "--inplace",
+    "convert_all": "--all",
+    "dtype": "--dtype",
+    "indices_dtype": "--indices-dtype",
+    "layout": "--layout",
+}
+
+#: Flags that only shape a recast; given without one of the above they have
+#: nothing to act on.
+_RECAST_ONLY_FLAGS = {"in_memory": "--in-memory", "chunk_rows": "--chunk"}
+
+#: Format-conversion flags, by parameter name, as `Options.given` spells them.
+_CONVERT_FLAGS = {
+    "zarr_format": "--zarr-format",
+    "compression": "--compression",
+    "force": "--force",
+    "overwrite": "--overwrite",
+    "genome": "--genome",
+    "var_names": "--var-names",
+    "make_unique": "--make-unique",
+    "feature_type": "--feature-type",
+    "split_feature_types": "--split-feature-types",
+    "layer": "--layer",
+    "gene_ids_column": "--gene-ids-column",
+    "gene_names_column": "--gene-names-column",
+    "feature_types_column": "--feature-types-column",
+}
+
+#: Flags that only make sense for format conversion; with recast's flags,
+#: they mean two commands' worth of request at once.
+_FORMAT_ONLY_FLAGS = {
+    "from_format": "--from",
+    "to_format": "--to",
+    **{
+        name: flag
+        for name, flag in _CONVERT_FLAGS.items()
+        if flag not in ("--force", "--zarr-format")
+    },
+}
+
+
+def _given(ctx: typer.Context, flags: dict) -> List[str]:
+    """The flags among `flags` that were actually typed, not defaulted."""
+    from click.core import ParameterSource
+
+    return [
+        flag
+        for name, flag in flags.items()
+        if ctx.get_parameter_source(name) not in (None, ParameterSource.DEFAULT)
+    ]
+
+
+@app.command("convert", context_settings={"allow_extra_args": True})
 def convert(
+    ctx: typer.Context,
+    file: Path = typer.Argument(
+        ...,
+        help="Input file or store",
+        exists=True,
+        readable=True,
+        dir_okay=True,
+        file_okay=True,
+    ),
+    output: Optional[Path] = typer.Option(
+        None,
+        "--output",
+        "-o",
+        help="Output path; its extension picks the format (.h5ad, .zarr, .h5)",
+        dir_okay=True,
+        file_okay=True,
+    ),
+    from_format: Optional[str] = typer.Option(
+        None,
+        "--from",
+        help=f"Input format, if it cannot be detected: {', '.join(FORMATS)}",
+    ),
+    to_format: Optional[str] = typer.Option(
+        None,
+        "--to",
+        help=f"Output format, if the extension does not say: {', '.join(FORMATS)}",
+    ),
+    zarr_format: Optional[int] = typer.Option(
+        None,
+        "--zarr-format",
+        help="Zarr spec version to write (defaults to the source store's, else 3)",
+    ),
+    compression: Optional[str] = typer.Option(
+        None,
+        "--compression",
+        help="none, gzip or lzf for h5ad; none, gzip, zstd or blosc for zarr "
+        "(defaults to the source's)",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        help="Convert despite a lossy or inflating result",
+    ),
+    overwrite: bool = typer.Option(
+        False,
+        "--overwrite",
+        help="Replace an existing output",
+    ),
+    genome: Optional[str] = typer.Option(
+        None,
+        "--genome",
+        help="10x input: the genome to keep (picks the group in a Cell Ranger 2 file)",
+    ),
+    var_names: str = typer.Option(
+        "gene_symbols",
+        "--var-names",
+        help="10x input: name features by gene_symbols or gene_ids",
+    ),
+    make_unique: bool = typer.Option(
+        False,
+        "--make-unique",
+        help="10x input: suffix repeated var names, as var_names_make_unique does",
+    ),
+    feature_type: Optional[str] = typer.Option(
+        None,
+        "--feature-type",
+        help="10x input: keep only this feature type, e.g. 'Gene Expression'",
+    ),
+    split_feature_types: bool = typer.Option(
+        False,
+        "--split-feature-types",
+        help="10x input: write one output per feature type, OUT.<type>.h5ad",
+    ),
+    layer: Optional[str] = typer.Option(
+        None,
+        "--layer",
+        help="10x output: the matrix to write: X (default), raw/X, or a layer",
+    ),
+    gene_ids_column: Optional[str] = typer.Option(
+        None,
+        "--gene-ids-column",
+        help="10x output: var column holding gene ids (default gene_ids)",
+    ),
+    gene_names_column: Optional[str] = typer.Option(
+        None,
+        "--gene-names-column",
+        help="10x output: var column holding gene names (default: var_names)",
+    ),
+    feature_types_column: Optional[str] = typer.Option(
+        None,
+        "--feature-types-column",
+        help="10x output: var column holding feature types (default feature_types)",
+    ),
+    # Deprecated: recast's flags, so `convert` keeps working as it did until
+    # 1.0.0. Hidden from --help; the old matrix paths arrive in ctx.args.
+    inplace: bool = typer.Option(False, "--inplace", hidden=True),
+    convert_all: bool = typer.Option(False, "--all", hidden=True),
+    dtype: Optional[str] = typer.Option(None, "--dtype", hidden=True),
+    indices_dtype: Optional[str] = typer.Option(None, "--indices-dtype", hidden=True),
+    layout: Optional[str] = typer.Option(None, "--layout", hidden=True),
+    in_memory: bool = typer.Option(False, "--in-memory", hidden=True),
+    chunk_rows: int = typer.Option(1024, "--chunk", "-C", hidden=True),
+) -> None:
+    """
+    Convert a store to another format.
+
+    The output's extension picks the format: .h5ad, .zarr, or .h5 for a 10x
+    Genomics matrix (Cell Ranger's *_feature_bc_matrix.h5). Between h5ad and
+    zarr every element is copied, streamed, keeping the source's chunking and
+    compression where the target can express them. A 10x matrix is read as
+    scanpy.read_10x_h5(gex_only=False) reads it, keeping integer counts, and
+    written as Cell Ranger writes it.
+
+    Nothing is written until every check has passed, and the output is moved
+    into place only once complete: a failed conversion leaves nothing
+    behind. An existing output is refused unless --overwrite is given.
+
+    To change a matrix's dtype, layout or density, use `recast`. Until 1.0.0,
+    `convert` given recast's flags still forwards to it, with a warning.
+
+    Examples:
+        adata convert data.h5ad -o data.zarr
+        adata convert data.zarr -o data.h5ad --compression gzip
+        adata convert data.zarr -o v3.zarr --zarr-format 3
+        adata convert filtered_feature_bc_matrix.h5 -o counts.h5ad
+        adata convert raw_feature_bc_matrix.h5 -o rna.h5ad --feature-type "Gene Expression"
+        adata convert counts.h5ad -o for_cellbender.h5 --layer counts
+    """
+    entries = list(ctx.args)
+    recast_flags = _given(ctx, _RECAST_FLAGS)
+
+    try:
+        if recast_flags:
+            mixed = _given(ctx, _FORMAT_ONLY_FLAGS)
+            if mixed:
+                raise ValueError(
+                    f"{', '.join(mixed)} cannot be combined with "
+                    f"{', '.join(recast_flags)}: the first change a store's "
+                    "format, the second its matrices. Run `adata recast` for "
+                    "the matrices."
+                )
+            console.print(
+                "[yellow]Warning:[/] 'convert' with "
+                f"{', '.join(recast_flags)} is deprecated and will be removed "
+                "in 1.0.0; use 'recast' instead.",
+            )
+            if not inplace and output is None:
+                raise ValueError("Output file is required. Use --output/-o or --inplace.")
+            if layout is not None and layout not in LAYOUTS:
+                raise ValueError(f"--layout must be one of: {', '.join(LAYOUTS)}.")
+            if zarr_format is not None and zarr_format not in (2, 3):
+                raise ValueError("--zarr-format must be 2 or 3.")
+            recast_store(
+                file=file,
+                entries=entries or None,
+                output=output,
+                console=console,
+                dtype=dtype,
+                indices_dtype=indices_dtype,
+                layout=layout,
+                convert_all=convert_all,
+                inplace=inplace,
+                chunk_rows=chunk_rows,
+                in_memory=in_memory,
+                force=force,
+                zarr_format=zarr_format,
+            )
+            return
+
+        if entries:
+            hint = (
+                f" To write it, pass -o {entries[0]}."
+                if output is None and len(entries) == 1
+                else ""
+            )
+            raise ValueError(
+                f"Unexpected argument{'s' if len(entries) > 1 else ''}: "
+                f"{' '.join(entries)}. convert takes one input.{hint} To change "
+                "a matrix's dtype or layout, use `adata recast`."
+            )
+        stray = _given(ctx, _RECAST_ONLY_FLAGS)
+        if stray:
+            raise ValueError(
+                f"{', '.join(stray)} {'applies' if len(stray) == 1 else 'apply'} "
+                "to `adata recast`, not to format conversion."
+            )
+        if output is None:
+            raise ValueError("Output path is required: pass -o/--output.")
+
+        convert_store(
+            file,
+            output,
+            console,
+            src_format=from_format,
+            dst_format=to_format,
+            options=Options(
+                given=frozenset(_given(ctx, _CONVERT_FLAGS)),
+                zarr_format=zarr_format,
+                compression=compression,
+                force=force,
+                overwrite=overwrite,
+                genome=genome,
+                var_names=var_names,
+                make_unique=make_unique,
+                feature_type=feature_type,
+                split_feature_types=split_feature_types,
+                layer=layer,
+                gene_ids_column=gene_ids_column,
+                gene_names_column=gene_names_column,
+                feature_types_column=feature_types_column,
+            ),
+        )
+    except Exception as e:
+        console.print(f"[bold red]Error:[/] {e}")
+        raise typer.Exit(code=1)
+
+
+# ============================================================================
+# RECAST command
+# ============================================================================
+@app.command("recast")
+def recast(
     file: Path = typer.Argument(
         ...,
         help="Input .h5ad/.zarr",
@@ -493,7 +772,7 @@ def convert(
     ),
     entries: Optional[List[str]] = typer.Argument(
         None,
-        help="Matrix paths to convert, e.g. 'X', 'layers/counts', 'raw/X'",
+        help="Matrix paths to recast, e.g. 'X', 'layers/counts', 'raw/X'",
     ),
     output: Optional[Path] = typer.Option(
         None,
@@ -511,7 +790,7 @@ def convert(
     convert_all: bool = typer.Option(
         False,
         "--all",
-        help="Convert X, every layer, and raw/X",
+        help="Recast X, every layer, and raw/X",
     ),
     dtype: Optional[str] = typer.Option(
         None,
@@ -531,7 +810,7 @@ def convert(
     force: bool = typer.Option(
         False,
         "--force",
-        help="Convert despite a lossy cast or a large size increase",
+        help="Recast despite a lossy cast or a large size increase",
     ),
     in_memory: bool = typer.Option(
         False,
@@ -562,11 +841,13 @@ def convert(
     both. Transposing streams by default, so it works on matrices too large
     to load.
 
+    This was `convert` until 0.7.0; `convert` now changes a store's format.
+
     Examples:
-        adata convert data.h5ad X -o out.h5ad --dtype float32
-        adata convert data.h5ad X -o out.h5ad --layout csc
-        adata convert data.h5ad X --inplace --dtype float32 --indices-dtype int32
-        adata convert data.h5ad --all -o out.h5ad --dtype float32
+        adata recast data.h5ad X -o out.h5ad --dtype float32
+        adata recast data.h5ad X -o out.h5ad --layout csc
+        adata recast data.h5ad X --inplace --dtype float32 --indices-dtype int32
+        adata recast data.h5ad --all -o out.h5ad --dtype float32
     """
     if not inplace and output is None:
         console.print(
@@ -586,7 +867,7 @@ def convert(
         raise typer.Exit(code=1)
 
     try:
-        convert_store(
+        recast_store(
             file=file,
             entries=list(entries) if entries else None,
             output=output,

adata/commands/__init__.py

@@ -6,4 +6,5 @@ from adata.commands.ls import list_store
 from adata.commands.create import create_store
 from adata.commands.split import split_store
 from adata.commands.concat import MERGE_CHOICES, MERGE_STRATEGIES, concat_stores
+from adata.commands.recast import recast_store
 from adata.commands.convert import convert_store

adata/commands/convert.py

@@ -1,218 +1,77 @@
-"""The `convert` command: rewrite matrices, copy everything else.
+"""The `convert` command: rewrite a store in another format.
 
-Writes a whole new store rather than editing one in place, for the same
-reason `subset` does: a conversion that fails half way leaves the original
-untouched, and `--inplace` becomes an atomic swap of a finished file rather
-than a partial edit of a live one.
+Every refusal happens while building the plan, before an output exists, and
+the outputs are written beside their destination and moved into place only
+once complete. A conversion that fails therefore leaves nothing behind, and
+never a half-written store that the next command would read as a whole one.
 """
 
 from __future__ import annotations
 
-import shutil
 from pathlib import Path
-from typing import Any, List, Optional
+from typing import List, Optional
 
 from rich.console import Console
 
-from adata.core.convert import (
-    DEFAULT_CHUNK,
-    LAYOUTS,
-    Plan,
-    convert_matrix,
-    parse_dtype,
-    plan_conversion,
+from adata.conversion import (
+    Options,
+    check_options,
+    converter_for,
+    detect_format,
+    output_format,
 )
-from adata.elements import spec
-from adata.elements.write import ensure_anndata_skeleton
-from adata.storage import copy_tree, detect_backend, is_group, open_store
-
-#: What `--all` means. The matrices an AnnData object is built from, not
-#: every 2-D array in the file: obsm/varm hold embeddings, whose dtype is
-#: rarely what anyone is trying to shrink. Those still convert by path.
-ALL_PREFIXES = ("X", "layers", "raw/X")
-
-
-def discover_matrices(root: Any) -> List[str]:
-    """Entry paths `--all` resolves to, in a stable order."""
-    found: List[str] = []
-    if "X" in root:
-        found.append("X")
-    if "layers" in root and is_group(root["layers"]):
-        found.extend(f"layers/{key}" for key in sorted(root["layers"].keys()))
-    if "raw" in root and is_group(root["raw"]) and "X" in root["raw"]:
-        found.append("raw/X")
-    return found
+from adata.util.staging import staged_outputs
 
 
 def _same_path(left: Path, right: Path) -> bool:
-    """Do these name the same store, through symlinks and `..` alike?"""
     try:
         return left.resolve() == right.resolve()
-    except OSError:  # pragma: no cover - unresolvable path
-        return left.absolute() == right.absolute()
-
-
-def _resolve(root: Any, path: str) -> Any:
-    obj = root
-    for part in path.split("/"):
-        if part not in obj:
-            raise KeyError(f"{path!r} not found in the store.")
-        obj = obj[part]
-    return obj
+    except OSError:
+        return False
 
 
 def convert_store(
     file: Path,
-    entries: Optional[List[str]],
-    output: Optional[Path],
+    output: Path,
     console: Console,
     *,
-    dtype: Optional[str] = None,
-    indices_dtype: Optional[str] = None,
-    layout: Optional[str] = None,
-    convert_all: bool = False,
-    inplace: bool = False,
-    chunk: int = DEFAULT_CHUNK,
-    chunk_rows: int = 1024,
-    in_memory: bool = False,
-    force: bool = False,
-    zarr_format: Optional[int] = None,
-) -> None:
-    """Write a copy of `file` with the named matrices converted."""
-    if not inplace and output is None:
-        raise ValueError("Output file is required unless --inplace is specified.")
-    if dtype is None and indices_dtype is None and layout is None:
-        raise ValueError(
-            "Nothing to do: pass at least one of --dtype, --indices-dtype "
-            "or --layout."
-        )
-    if layout is not None and layout not in LAYOUTS:
-        raise ValueError(f"--layout must be one of: {', '.join(LAYOUTS)}.")
-
-    data_dtype = parse_dtype(dtype) if dtype else None
-    index_dtype = (
-        parse_dtype(indices_dtype, allowed=("int32", "int64"))
-        if indices_dtype
-        else None
-    )
-
-    if output is not None and not inplace and _same_path(file, output):
-        # Opening the destination "w" clears it while the source is still
-        # being read from it. HDF5 refuses; Zarr does not, and quietly
-        # produced an empty store where the data used to be.
+    src_format: Optional[str] = None,
+    dst_format: Optional[str] = None,
+    options: Options = Options(),
+) -> List[Path]:
+    """Convert `file` into `output`; return the paths written."""
+    file, output = Path(file), Path(output)
+    if str(output) == "-":
         raise ValueError(
-            f"Output path is the input: {output}. Use --inplace to replace "
-            "it, which writes to a temporary file first."
-        )
-
-    if inplace:
-        backend = detect_backend(file)
-        if backend == "zarr":
-            base = file.stem if file.suffix else file.name
-            dst_path = file.with_name(f"{base}.convert-tmp.zarr")
-        else:
-            dst_path = file.with_name(f"{file.name}.convert-tmp")
-        if dst_path.exists():
-            raise FileExistsError(f"Temporary path already exists: {dst_path}")
-    else:
-        dst_path = output
-
-    if zarr_format is None and detect_backend(file) == "zarr":
-        with open_store(file, "r") as probe:
-            zarr_format = probe.zarr_format
-
-    with open_store(file, "r") as src_store:
-        src = src_store.root
-        targets = discover_matrices(src) if convert_all else list(entries or [])
-        if not targets:
-            raise ValueError(
-                "No matrices selected. Name one (e.g. `X`) or pass --all."
-            )
-        # Every check first, before the destination exists. A refusal must
-        # not leave behind a store holding a copy of obs and var and nothing
-        # else -- the caller cannot tell that from a finished conversion.
-        plans = {
-            path: plan_conversion(
-                _resolve(src, path),
-                path,
-                dtype=data_dtype,
-                index_dtype=index_dtype,
-                layout=layout,
-                chunk=chunk,
-                force=force,
-                console=console,
-            )
-            for path in targets
-        }
-
-        console.print(
-            f"[cyan]Converting {len(targets)} matrix/matrices:[/] "
-            + ", ".join(targets)
+            "convert writes a store, which cannot go to stdout; give a path "
+            "with -o."
         )
-
-        with open_store(dst_path, "w", zarr_format=zarr_format) as dst_store:
-            dst = dst_store.root
-            _write(
-                src, dst, plans,
-                chunk=chunk, chunk_rows=chunk_rows, in_memory=in_memory,
-                console=console,
-            )
-            ensure_anndata_skeleton(dst)
-
-    if inplace:
-        if file.is_dir():
-            shutil.rmtree(file)
-        elif file.exists():
-            file.unlink()
-        if dst_path.is_dir():
-            shutil.move(str(dst_path), str(file))
-        else:
-            dst_path.replace(file)
-        console.print(f"[green]Converted[/] {file}")
-    else:
-        console.print(f"[green]Wrote[/] {dst_path}")
-
-
-def _write(src: Any, dst: Any, plans: dict, **options: Any) -> None:
-    """Copy the store across, converting the targeted entries as they pass."""
-    by_parent: dict = {}
-    for path in plans:
-        parent, _, leaf = path.rpartition("/")
-        by_parent.setdefault(parent, {})[leaf] = plans[path]
-
-    for key in src.keys():
-        if key in by_parent.get("", {}):
-            convert_matrix(
-                src[key], dst, key, plan=by_parent[""][key], **options
+    if options.zarr_format not in (None, 2, 3):
+        raise ValueError("--zarr-format must be 2 or 3.")
+
+    src_format = src_format or detect_format(file)
+    dst_format = output_format(output, dst_format)
+    converter = converter_for(src_format, dst_format)
+    check_options(converter, options, src_format, dst_format)
+
+    plan = converter.plan(file, output, src_format, dst_format, options)
+    for out in plan.outputs:
+        if _same_path(file, out):
+            # Writing a Zarr store over its own source empties it before a
+            # single element has been read.
+            raise ValueError(f"Output path is the input: {out}")
+        if out.exists() and not options.overwrite:
+            raise FileExistsError(
+                f"'{out}' already exists. Pass --overwrite to replace it."
             )
-        elif key in by_parent:
-            # Any parent, not just layers and raw. Restricting it to those
-            # two meant an explicitly named `obsm/X_pca` was copied
-            # unconverted and the command still reported success.
-            _write_group(src[key], dst, key, by_parent, **options)
-        else:
-            copy_tree(src[key], dst, key)
-
-
-def _write_group(
-    group: Any, dst: Any, name: str, by_parent: dict, **options: Any
-) -> None:
-    """Recreate one container, converting the members that were named."""
-    from adata.storage import copy_attrs, is_zarr_group
 
-    out = dst.create_group(name)
-    copy_attrs(
-        group.attrs,
-        out.attrs,
-        target_backend="zarr" if is_zarr_group(dst) else "hdf5",
-    )
-    if not spec.encoding_type(out):
-        spec.set_encoding(out, spec.RAW if name == "raw" else spec.DICT)
+    console.print(f"[cyan]Converting[/] {file}: {plan.summary}")
+    for warning in plan.warnings:
+        console.print(f"[yellow]Warning:[/] {warning}")
 
+    with staged_outputs(plan.outputs) as staged:
+        converter.write(plan, staged, console)
 
-    wanted = by_parent.get(name, {})
-    for key in group.keys():
-        if key in wanted:
-            convert_matrix(group[key], out, key, plan=wanted[key], **options)
-        else:
-            copy_tree(group[key], out, key)
+    for out in plan.outputs:
+        console.print(f"[green]Wrote[/] {out}")
+    return plan.outputs