pyadata-cli 0.5.0__py3-none-any.whl

adata/formats/image.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+from PIL import Image
+from rich.console import Console
+
+from adata.elements.write import write_dense, write_mapping
+from adata.formats.validate import DATAFRAME_PATHS, validate_dimensions
+from adata.formats.common import _resolve
+from adata.storage import is_dataset
+from adata.util.path import norm_path
+
+
+def export_image(root: Any, obj: str, out: Path, console: Console) -> None:
+    h5obj = _resolve(root, obj)
+    if not is_dataset(h5obj):
+        raise ValueError("Image export requires a dataset.")
+    arr = np.asarray(h5obj[...])
+
+    if arr.ndim not in (2, 3):
+        raise ValueError(f"Expected 2D or 3D image array; got shape {arr.shape}.")
+    if arr.ndim == 3 and arr.shape[2] not in (1, 3, 4):
+        raise ValueError(
+            f"Expected last dimension (channels) to be 1, 3, or 4; got {arr.shape}."
+        )
+
+    if np.issubdtype(arr.dtype, np.floating):
+        amax = float(np.nanmax(arr)) if arr.size else 0.0
+        if amax <= 1.0:
+            arr = np.clip(arr, 0.0, 1.0) * 255.0
+        else:
+            arr = np.clip(arr, 0.0, 255.0)
+        arr = arr.astype(np.uint8)
+    elif np.issubdtype(arr.dtype, np.integer):
+        arr = np.clip(arr, 0, 255).astype(np.uint8)
+    elif arr.dtype == np.bool_:
+        arr = arr.astype(np.uint8) * 255
+    else:
+        raise ValueError(f"Unsupported image dtype: {arr.dtype}")
+
+    if arr.ndim == 3 and arr.shape[2] == 1:
+        arr = arr[:, :, 0]
+
+    img = Image.fromarray(arr)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    img.save(out)
+    console.print(f"[green]Wrote[/] {out}")
+
+
+def import_image(root: Any, obj: str, input_file: Path, console: Console) -> None:
+    """Read an image file into the store as a dense array.
+
+    Kept as raw pixels (H, W) or (H, W, C) with no encoding beyond `array`,
+    which is how spatial tooling stores tissue images under uns/spatial.
+    """
+    obj = norm_path(obj)
+    arr = np.asarray(Image.open(input_file))
+
+    if arr.ndim not in (2, 3):
+        raise ValueError(f"Expected a 2D or 3D image; got shape {arr.shape}.")
+
+    if obj in DATAFRAME_PATHS:
+        raise ValueError(
+            f"'{obj}' must hold a dataframe; writing an image there would "
+            "corrupt the store. Images belong somewhere unstructured, such as "
+            "'uns/spatial/hires'."
+        )
+
+    # An image's height is not an axis, so a path that implies one is almost
+    # certainly a mistake -- but check it rather than assume.
+    validate_dimensions(root, obj, arr.shape, console)
+
+    parts = obj.split("/")
+    parent = root
+    for part in parts[:-1]:
+        parent = parent[part] if part in parent else write_mapping(parent, part)
+
+    write_dense(parent, parts[-1], arr, replace=True)
+    console.print(
+        f"[green]Imported[/] {'x'.join(str(d) for d in arr.shape)} "
+        f"image ({arr.dtype}) into '{obj}'"
+    )

adata/formats/json_data.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+import numpy as np
+from rich.console import Console
+
+from adata.core.read import decode_str_array
+from adata.elements import spec
+from adata.formats.common import _check_json_exportable, _resolve
+from adata.elements.write import (
+    write_dense,
+    write_mapping,
+    write_null,
+    write_scalar,
+    write_string_array,
+)
+from adata.storage import create_dataset, is_dataset, is_group
+from adata.util.path import norm_path
+
+
+def export_json(
+    root: Any,
+    obj: str,
+    out: Path | None,
+    max_elements: int,
+    include_attrs: bool,
+    console: Console,
+) -> None:
+    h5obj = _resolve(root, obj)
+    _check_json_exportable(h5obj, max_elements=max_elements)
+
+    payload = _to_jsonable(
+        h5obj, max_elements=max_elements, include_attrs=include_attrs
+    )
+    if out is None or str(out) == "-":
+        out_fh = sys.stdout
+    else:
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out_fh = open(out, "w", encoding="utf-8")
+    try:
+        json.dump(payload, out_fh, indent=2, ensure_ascii=False, sort_keys=True)
+        out_fh.write("\n")
+    finally:
+        if out_fh is not sys.stdout:
+            out_fh.close()
+    if out_fh is not sys.stdout:
+        console.print(f"[green]Wrote[/] {out}")
+
+
+def _attrs_to_jsonable(attrs: Any, max_elements: int) -> Dict[str, Any]:
+    out: Dict[str, Any] = {}
+    for k in attrs.keys():
+        v = attrs.get(k)
+        out[str(k)] = _pyify(v, max_elements=max_elements)
+    return out
+
+
+def _pyify(value: Any, max_elements: int) -> Any:
+    if isinstance(value, bytes):
+        try:
+            return value.decode("utf-8")
+        except Exception:
+            return value.decode("utf-8", errors="replace")
+    if isinstance(value, np.generic):
+        return value.item()
+    if isinstance(value, np.ndarray):
+        if value.size > max_elements:
+            raise ValueError(
+                f"Refusing to convert array of size {value.size} (> {max_elements}) to JSON."
+            )
+        if np.issubdtype(value.dtype, np.bytes_) or value.dtype.kind == "O":
+            value = decode_str_array(value)
+        return value.tolist()
+    return value
+
+
+def _dataset_to_jsonable(ds: Any, max_elements: int) -> Any:
+    if spec.encoding_type(ds) == spec.NULL:
+        # A null is a placeholder -- an h5py.Empty or a 0-d zarr bool -- whose
+        # stored value is meaningless. Reading it would emit that placeholder
+        # rather than restoring the None it represents.
+        return None
+
+    if ds.shape == ():
+        v = ds[()]
+        return _pyify(v, max_elements=max_elements)
+    n = int(np.prod(ds.shape)) if ds.shape else 0
+    if n > max_elements:
+        ds_name = getattr(ds, "name", "<dataset>")
+        raise ValueError(
+            f"Refusing to convert dataset {ds_name!r} with {n} elements (> {max_elements}) to JSON."
+        )
+    arr = np.asarray(ds[...])
+    return _pyify(arr, max_elements=max_elements)
+
+
+def _to_jsonable(h5obj: Any, max_elements: int, include_attrs: bool) -> Any:
+    if is_dataset(h5obj):
+        return _dataset_to_jsonable(h5obj, max_elements=max_elements)
+
+    d: Dict[str, Any] = {}
+    if include_attrs and len(h5obj.attrs):
+        d["__attrs__"] = _attrs_to_jsonable(h5obj.attrs, max_elements=max_elements)
+
+    for key in h5obj.keys():
+        child = h5obj[key]
+        if is_group(child) or is_dataset(child):
+            d[str(key)] = _to_jsonable(
+                child,
+                max_elements=max_elements,
+                include_attrs=include_attrs,
+            )
+    return d
+
+
+def import_json(
+    root: Any,
+    obj: str,
+    input_file: Path,
+    console: Console,
+) -> None:
+    obj = norm_path(obj)
+    with open(input_file, "r", encoding="utf-8") as fh:
+        payload = json.load(fh)
+
+    parts = obj.split("/")
+    parent = root
+    for part in parts[:-1]:
+        parent = parent[part] if part in parent else parent.create_group(part)
+    name = parts[-1]
+
+    if name in parent:
+        del parent[name]
+
+    _write_json_to_group(parent, name, payload)
+
+    console.print(f"[green]Imported[/] JSON data into '{obj}'")
+
+
+def _write_json_to_group(parent: Any, name: str, value: Any) -> None:
+    """Write one JSON value as the AnnData element that best represents it."""
+    if isinstance(value, dict):
+        group = write_mapping(parent, name, replace=True)
+        for k, v in value.items():
+            _write_json_to_group(group, k, v)
+        return
+
+    if value is None:
+        write_null(parent, name, replace=True)
+        return
+
+    if isinstance(value, str):
+        write_scalar(parent, name, value, replace=True)
+        return
+
+    if isinstance(value, (bool, int, float)):
+        write_scalar(parent, name, value, replace=True)
+        return
+
+    if isinstance(value, list):
+        _write_json_list(parent, name, value)
+        return
+
+    raise ValueError(f"Cannot convert JSON value of type {type(value).__name__}")
+
+
+def _write_json_list(parent: Any, name: str, value: list) -> None:
+    """Write a JSON array as a string or numeric array where it is uniform.
+
+    Ragged or mixed lists have no array representation in the spec, so they
+    are stored as their JSON text rather than silently reshaped.
+    """
+    if all(isinstance(v, str) for v in value):
+        write_string_array(parent, name, value, replace=True)
+        return
+
+    try:
+        arr = np.array(value)
+    except (ValueError, TypeError):
+        arr = None
+
+    if arr is not None and arr.dtype.kind in ("b", "i", "u", "f"):
+        write_dense(parent, name, arr, replace=True)
+        return
+
+    if arr is not None and arr.dtype.kind in ("U", "S", "O", "T"):
+        # Pass the shaped array through: flattening would turn a nested list
+        # such as [["a","b"],["c","d"]] into a length-4 vector.
+        write_string_array(parent, name, arr, replace=True)
+        return
+
+    write_scalar(parent, name, json.dumps(value), replace=True)

adata/formats/sparse.py

@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, List, Tuple
+import sys
+from contextlib import nullcontext
+
+import numpy as np
+from rich.console import Console
+
+from adata.elements import spec
+from adata.elements.write import write_sparse
+from adata.formats.common import _get_encoding_type, _resolve
+from adata.formats.validate import validate_dimensions
+from adata.storage import create_dataset, is_dataset, is_group, is_zarr_group
+from adata.util.path import norm_path
+
+
+def _read_mtx(
+    input_file: Path,
+) -> Tuple[List[Tuple[int, int, float]], Tuple[int, int], int]:
+    with open(input_file, "r", encoding="utf-8") as fh:
+        header = fh.readline()
+        if not header.startswith("%%MatrixMarket"):
+            raise ValueError("Invalid MTX file: missing MatrixMarket header.")
+
+        parts = header.lower().split()
+        field = "real"
+        for p in parts:
+            if p in ("real", "integer", "complex", "pattern"):
+                field = p
+                break
+
+        line = fh.readline()
+        while line.startswith("%"):
+            line = fh.readline()
+
+        dims = line.split()
+        n_rows, n_cols, nnz = int(dims[0]), int(dims[1]), int(dims[2])
+
+        entries = []
+        for _ in range(nnz):
+            parts = fh.readline().split()
+            r, c = int(parts[0]) - 1, int(parts[1]) - 1
+            if field == "pattern":
+                v = 1.0
+            else:
+                v = float(parts[2])
+            entries.append((r, c, v))
+
+    return entries, (n_rows, n_cols), nnz
+
+
+def _create_csr_from_entries(
+    entries: List[Tuple[int, int, float]], shape: Tuple[int, int]
+) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
+    n_rows, _ = shape
+    entries.sort(key=lambda x: (x[0], x[1]))
+
+    data = np.array([e[2] for e in entries], dtype=np.float32)
+    indices = np.array([e[1] for e in entries], dtype=np.int32)
+
+    indptr = np.zeros(n_rows + 1, dtype=np.int32)
+    for r, _, _ in entries:
+        indptr[r + 1] += 1
+    indptr = np.cumsum(indptr)
+
+    return data, indices, indptr
+
+
+def export_mtx(
+    root: Any,
+    obj: str,
+    out: Path | None,
+    head: int | None,
+    chunk_elements: int,
+    in_memory: bool,
+    console: Console,
+) -> None:
+    h5obj = _resolve(root, obj)
+    if not is_group(h5obj):
+        raise ValueError("MTX export requires a CSR/CSC matrix group (not a dataset).")
+
+    enc = _get_encoding_type(h5obj)
+    if enc not in ("csr_matrix", "csc_matrix"):
+        raise ValueError(
+            f"Target group encoding-type is {enc!r}; expected 'csr_matrix' or 'csc_matrix'."
+        )
+
+    data = h5obj.get("data")
+    indices = h5obj.get("indices")
+    indptr = h5obj.get("indptr")
+    if not (is_dataset(data) and is_dataset(indices) and is_dataset(indptr)):
+        raise ValueError(
+            "Sparse matrix group must contain datasets: data, indices, indptr"
+        )
+
+    shape = h5obj.attrs.get("shape", None)
+    if shape is None:
+        raise ValueError("Sparse matrix group is missing required 'shape' attribute.")
+    n_rows, n_cols = (int(shape[0]), int(shape[1]))
+
+    field = "real" if np.issubdtype(data.dtype, np.floating) else "integer"
+
+    indptr_arr = np.asarray(indptr[...], dtype=np.int64)
+    nnz_ptr = int(indptr_arr[-1]) if indptr_arr.size else 0
+    nnz_data = int(data.shape[0])
+    nnz_idx = int(indices.shape[0])
+
+    if not (nnz_ptr == nnz_data == nnz_idx):
+        raise ValueError(
+            f"Sparse matrix data inconsistency: indptr implies {nnz_ptr} nonzeros, "
+            f"but data has {nnz_data} and indices has {nnz_idx}."
+        )
+
+    nnz = nnz_data
+    major_step = max(1, int(chunk_elements))
+    if head is not None and head > 0:
+        nnz = min(nnz_data, head)
+
+    if out is None or str(out) == "-":
+        out_fh = sys.stdout
+    else:
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out_fh = open(out, "w", encoding="utf-8", newline="\n")
+
+    use_status = out_fh is not sys.stdout
+    status_ctx = (
+        console.status(f"[magenta]Exporting {obj} to {out}...[/]")
+        if use_status
+        else nullcontext()
+    )
+    try:
+        out_fh.write(f"%%MatrixMarket matrix coordinate {field} general\n")
+        out_fh.write("% generated by h5ad-cli\n")
+        if head is not None and head > 0:
+            out_fh.write(f"% output limited to first {nnz}/{nnz_data} nonzero entries\n")
+        out_fh.write(f"{n_rows} {n_cols} {nnz}\n")
+
+        if in_memory:
+            with status_ctx as status:
+                if use_status and status:
+                    status.update(
+                        f"[magenta]Loading entire matrix {obj} into memory...[/]"
+                    )
+                data_arr = np.asarray(data[...])
+                indices_arr = np.asarray(indices[...], dtype=np.int64)
+                counts = np.diff(indptr_arr)
+                if int(counts.sum()) != nnz_data:
+                    raise ValueError(
+                        "Sparse matrix indptr does not match data/indices length."
+                    )
+
+                if enc == "csr_matrix":
+                    major_idx = np.repeat(np.arange(n_rows, dtype=np.int64), counts)
+                    row_idx = major_idx
+                    col_idx = indices_arr
+                else:
+                    major_idx = np.repeat(np.arange(n_cols, dtype=np.int64), counts)
+                    row_idx = indices_arr
+                    col_idx = major_idx
+
+                if head is not None and head > 0:
+                    row_idx = row_idx[:nnz]
+                    col_idx = col_idx[:nnz]
+                    data_arr = data_arr[:nnz]
+
+                data_fmt = "%.18g" if field == "real" else "%d"
+                coords = np.column_stack((row_idx + 1, col_idx + 1, data_arr))
+                if use_status and status:
+                    status.update(f"[magenta]Saving {nnz} entries to {out}...[/]")
+                np.savetxt(out_fh, coords, fmt=["%d", "%d", data_fmt], newline="\n")
+        else:
+            major = n_rows if enc == "csr_matrix" else n_cols
+            max_lines = head if head is not None and head > 0 else None
+            written = 0
+            with status_ctx as status:
+                for major_start in range(0, major, major_step):
+                    major_end = min(major_start + major_step, major)
+                    if use_status and status:
+                        status.update(
+                            f"[magenta]Exporting {obj}: {major_start+1}-{major_end} of {major}...[/]"
+                        )
+                    for major_i in range(major_start, major_end):
+                        start = min(int(indptr_arr[major_i]), nnz_data)
+                        end = min(int(indptr_arr[major_i + 1]), nnz_data)
+                        if end <= start:
+                            continue
+                        idx = np.asarray(indices[start:end], dtype=np.int64)
+                        vals = np.asarray(data[start:end])
+                        m = min(len(idx), len(vals))
+                        if m == 0:
+                            raise ValueError("Sparse matrix chunk has zero length.")
+                        if max_lines is not None:
+                            remaining = max_lines - written
+                            if remaining <= 0:
+                                break
+                            if m > remaining:
+                                m = remaining
+                        idx = idx[:m]
+                        vals = vals[:m]
+                        idx_list = idx.tolist()
+                        vals_list = vals.tolist()
+                        if enc == "csr_matrix":
+                            r = major_i + 1
+                            lines = [
+                                f"{r} {c + 1} {v}\n"
+                                for c, v in zip(idx_list, vals_list)
+                            ]
+                        else:
+                            c = major_i + 1
+                            lines = [
+                                f"{r + 1} {c} {v}\n"
+                                for r, v in zip(idx_list, vals_list)
+                            ]
+                        out_fh.write("".join(lines))
+                        written += m
+                        if max_lines is not None and written >= max_lines:
+                            break
+                    if max_lines is not None and written >= max_lines:
+                        break
+    finally:
+        if out_fh is not sys.stdout:
+            out_fh.close()
+    if out_fh is not sys.stdout:
+        console.print(f"[green]Wrote[/] {out}")
+
+
+def import_mtx(
+    root: Any,
+    obj: str,
+    input_file: Path,
+    console: Console,
+) -> None:
+    obj = norm_path(obj)
+    entries, shape, nnz = _read_mtx(input_file)
+    data, indices, indptr = _create_csr_from_entries(entries, shape)
+
+    validate_dimensions(root, obj, shape, console)
+
+    parts = obj.split("/")
+    parent = root
+    for part in parts[:-1]:
+        parent = parent[part] if part in parent else parent.create_group(part)
+    name = parts[-1]
+
+    write_sparse(
+        parent, name, data, indices, indptr, shape, spec.CSR_MATRIX, replace=True
+    )
+
+    console.print(
+        f"[green]Imported[/] {shape[0]}×{shape[1]} sparse matrix ({nnz} non-zero) into '{obj}'"
+    )