pyadata-cli 0.6.0__tar.gz → 0.6.1__tar.gz

{pyadata_cli-0.6.0 → pyadata_cli-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyadata-cli
-Version: 0.6.0
+Version: 0.6.1
 Summary: Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores
 Keywords: anndata,h5ad,zarr,bioinformatics,single-cell,cli
 Author: Aljes Binkevich

{pyadata_cli-0.6.0 → pyadata_cli-0.6.1}/pyproject.toml

@@ -3,7 +3,7 @@
 # so the distribution is published as `pyadata-cli`. The import package
 # and the command are both still `adata`.
 name = "pyadata-cli"
-version = "0.6.0"
+version = "0.6.1"
 description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores"
 readme = "README.md"
 requires-python = ">=3.12"

{pyadata_cli-0.6.0 → pyadata_cli-0.6.1}/src/adata/cli.py

@@ -26,7 +26,7 @@ from adata.commands import export_image as export_image_cmd
 
 app = typer.Typer(
     help="Streaming CLI for huge AnnData .h5ad and .zarr stores "
-    "(view, ls, subset, split, concat, export, import)."
+    "(view, ls, create, subset, split, concat, convert, export, import)."
 )
 # Use stderr for status/progress to keep stdout clean for data output
 # force_terminal=True ensures Rich output is visible even in non-TTY environments

{pyadata_cli-0.6.0 → pyadata_cli-0.6.1}/src/adata/commands/split.py

@@ -3,7 +3,7 @@
 Requested in issue #2, modelled on cellgeni/scraft's `split_h5ad` but
 streaming: the source is never loaded into memory, only the column being split
 on is read, and each output is produced by the same subset machinery used by
-`adata subset`.
+`adata subset`, fanned out so every matrix is read once for all groups.
 """
 
 from __future__ import annotations
@@ -17,7 +17,7 @@ import numpy as np
 from rich.console import Console
 
 from adata.core.select import group_indices
-from adata.core.subset import subset_h5ad
+from adata.core.subset import split_h5ad
 from adata.storage import detect_backend, open_store
 
 
@@ -112,19 +112,22 @@ def split_store(
 
     output_dir.mkdir(parents=True, exist_ok=True)
 
-    for label, out_path, _ in planned:
-        indices = np.sort(groups[label])
-        subset_h5ad(
-            file=file,
-            output=out_path,
-            obs_file=None,
-            var_file=None,
-            chunk_rows=chunk_rows,
-            console=console,
-            obs_indices=indices if axis == "obs" else None,
-            var_indices=indices if axis == "var" else None,
-            zarr_format=zarr_format,
-        )
+    # One call for every group, so each matrix is read once rather than once
+    # per group.
+    split_h5ad(
+        file,
+        [
+            (
+                out_path,
+                np.sort(groups[label]) if axis == "obs" else None,
+                np.sort(groups[label]) if axis == "var" else None,
+            )
+            for label, out_path, _ in planned
+        ],
+        chunk_rows=chunk_rows,
+        console=console,
+        zarr_format=zarr_format,
+    )
 
     if manifest:
         _write_manifest(file, output_dir, column, planned, console)

{pyadata_cli-0.6.0 → pyadata_cli-0.6.1}/src/adata/core/concat.py

@@ -389,6 +389,45 @@ def _matrix_kind(obj: Any) -> str:
     return "other"
 
 
+def _widen_to_hold(dtype: np.dtype, largest: int) -> np.dtype:
+    """`dtype`, or the narrowest of int32/int64 wider than it that holds `largest`."""
+    dtype = np.dtype(dtype)
+    if largest <= np.iinfo(dtype).max:
+        return dtype
+    for candidate in (np.int32, np.int64):
+        if largest <= np.iinfo(candidate).max:
+            return np.dtype(np.result_type(dtype, candidate))
+    return np.dtype(np.int64)
+
+
+def _concat_index_dtypes(
+    sources: List[Any], n_minor: int
+) -> Tuple[np.dtype, np.dtype]:
+    """The `indices` and `indptr` dtypes for a concatenation of `sources`.
+
+    The widest the inputs used, so int32 inputs give an int32 output rather
+    than the int64 every output used to get. That is widened only when the
+    result could not be addressed otherwise: `indices` must reach the
+    output's minor dimension, and `indptr` its total nonzero count, which is
+    known before writing as the sum of the inputs'. Both are checked against
+    the chosen dtype's own limit, not int32's, since the spec allows
+    narrower index arrays and an int16 `indptr` would otherwise wrap.
+    """
+    indices = np.result_type(*[s["indices"].dtype for s in sources])
+    indptr = np.result_type(*[s["indptr"].dtype for s in sources])
+    nnz = sum(int(s["data"].shape[0]) for s in sources)
+    return _widen_to_hold(indices, n_minor), _widen_to_hold(indptr, nnz)
+
+
+def _write_indptr(
+    group: Any, indptr: List[int], dtype: np.dtype, template: Any
+) -> None:
+    from adata.core.subset import _sparse_dataset
+
+    values = np.asarray(indptr, dtype=np.int64).astype(dtype, copy=False)
+    _sparse_dataset(group, "indptr", dtype, values.size, template)[:] = values
+
+
 def _concat_sparse(
     dst_parent: Any,
     name: str,
@@ -403,7 +442,7 @@ def _concat_sparse(
     Absent columns need no fill: a sparse matrix's zeros are implicit, so
     dropped entries simply do not appear in the output.
     """
-    from adata.core.subset import _append, _growable
+    from adata.core.subset import _append, _growable_like
 
     group = dst_parent.create_group(name)
     spec.set_encoding(group, spec.CSR_MATRIX)
@@ -412,8 +451,11 @@ def _concat_sparse(
     set_shape_attr(group, (n_rows, n_cols))
 
     dtype = np.result_type(*[s["data"].dtype for s in sources])
-    out_data = _growable(group, "data", dtype)
-    out_indices = _growable(group, "indices", np.int64)
+    index_dtype, pointer_dtype = _concat_index_dtypes(sources, n_cols)
+    out_data = _growable_like(group, "data", dtype, sources[0]["data"])
+    out_indices = _growable_like(
+        group, "indices", index_dtype, sources[0]["indices"]
+    )
     indptr = [0]
     nnz = 0
 
@@ -449,7 +491,7 @@ def _concat_sparse(
                 _append(out_indices, np.concatenate(kept_idx))
                 _append(out_data, np.concatenate(kept_data).astype(dtype))
 
-    create_dataset(group, "indptr", data=np.asarray(indptr, dtype=np.int64))
+    _write_indptr(group, indptr, pointer_dtype, sources[0]["indptr"])
 
 
 def _concat_dense(
@@ -509,7 +551,7 @@ def _concat_csc(
     input's row offset added. Because every input's column is already sorted
     and the offsets increase, the result is sorted without a re-sort.
     """
-    from adata.core.subset import _append, _growable
+    from adata.core.subset import _append, _growable_like
     from adata.elements.write import set_shape_attr
 
     group = dst_parent.create_group(name)
@@ -517,8 +559,11 @@ def _concat_csc(
     set_shape_attr(group, (n_rows, n_cols))
 
     dtype = np.result_type(*[s["data"].dtype for s in sources])
-    out_data = _growable(group, "data", dtype)
-    out_indices = _growable(group, "indices", np.int64)
+    index_dtype, pointer_dtype = _concat_index_dtypes(sources, n_rows)
+    out_data = _growable_like(group, "data", dtype, sources[0]["data"])
+    out_indices = _growable_like(
+        group, "indices", index_dtype, sources[0]["indices"]
+    )
     indptr = [0]
     nnz = 0
 
@@ -547,7 +592,7 @@ def _concat_csc(
             nnz += int(sum(len(r) for r in rows))
         indptr.append(nnz)
 
-    create_dataset(group, "indptr", data=np.asarray(indptr, dtype=np.int64))
+    _write_indptr(group, indptr, pointer_dtype, sources[0]["indptr"])
 
 
 def check_matrix_encodings(roots: List[Any], console: Console) -> None:

{pyadata_cli-0.6.0 → pyadata_cli-0.6.1}/src/adata/core/convert.py

@@ -31,6 +31,7 @@ from rich.console import Console
 
 from adata.elements import spec
 from adata.elements.write import set_shape_attr
+from adata.core.subset import _growable_like, _sparse_dataset
 from adata.storage import (
     copy_attrs,
     create_dataset,
@@ -298,41 +299,6 @@ def _new_sparse_group(
     return group
 
 
-def _sparse_dataset(
-    group: Any, name: str, dtype: np.dtype, n: int, template: Any
-) -> Any:
-    """A 1-D dataset of `n` elements laid out like `template`.
-
-    Forwarding compression and chunking matters more here than anywhere
-    else: the point of a dtype change is usually to make the file smaller,
-    and creating the output with a fixed 65,536-element chunk made a
-    2,400-nonzero matrix allocate 786 KB of mostly empty chunk -- eight
-    times the source, from a conversion asked for to halve it.
-    """
-    from adata.core.subset import _clamp_chunks
-
-    backend = "zarr" if is_zarr_group(group) else "hdf5"
-    kw = dataset_create_kwargs(template, target_backend=backend, dst_parent=group)
-    kw = _clamp_chunks(kw, max(1, n))
-    if "chunks" not in kw and n:
-        kw["chunks"] = (min(n, 1 << 16),)
-    return create_dataset(group, name, shape=(n,), dtype=dtype, **kw)
-
-
-def _growable_like(group: Any, name: str, dtype: np.dtype, template: Any) -> Any:
-    """Like `_sparse_dataset`, but extensible for a size not yet known."""
-    backend = "zarr" if is_zarr_group(group) else "hdf5"
-    kw = dataset_create_kwargs(template, target_backend=backend, dst_parent=group)
-    kw.pop("shards", None)
-    chunks = kw.pop("chunks", None)
-    step = int(chunks[0]) if chunks else 1 << 16
-    if is_zarr_group(group):
-        return group.create_array(name, shape=(0,), dtype=dtype, chunks=(step,), **kw)
-    return group.create_dataset(
-        name, shape=(0,), maxshape=(None,), dtype=dtype, chunks=(step,), **kw
-    )
-
-
 def _write_sparse_arrays(
     group: Any,
     data: np.ndarray,